From 20a33efdf32bf0aedcb0c9813ddc7572bb1ab8c7 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 18 Feb 2022 13:10:34 -0800 Subject: libgo: update to Go1.18rc1 release Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/386594 --- libgo/go/cmd/go/alldocs.go | 38 +-- libgo/go/cmd/go/go_test.go | 2 +- libgo/go/cmd/go/internal/base/flag.go | 7 - libgo/go/cmd/go/internal/cfg/cfg.go | 6 +- libgo/go/cmd/go/internal/envcmd/env.go | 4 + libgo/go/cmd/go/internal/help/helpdoc.go | 10 + libgo/go/cmd/go/internal/list/list.go | 1 - libgo/go/cmd/go/internal/load/pkg.go | 22 +- libgo/go/cmd/go/internal/modcmd/download.go | 1 - libgo/go/cmd/go/internal/modcmd/graph.go | 1 - libgo/go/cmd/go/internal/modcmd/verify.go | 1 - libgo/go/cmd/go/internal/modcmd/why.go | 1 - libgo/go/cmd/go/internal/modfetch/coderepo.go | 216 +++++++-------- libgo/go/cmd/go/internal/modfetch/coderepo_test.go | 301 ++++++++++++--------- libgo/go/cmd/go/internal/modfetch/fetch.go | 11 +- libgo/go/cmd/go/internal/modload/import.go | 18 +- libgo/go/cmd/go/internal/modload/init.go | 35 ++- libgo/go/cmd/go/internal/modload/load.go | 30 +- libgo/go/cmd/go/internal/run/run.go | 1 - libgo/go/cmd/go/internal/test/testflag.go | 1 - libgo/go/cmd/go/internal/version/version.go | 9 +- libgo/go/cmd/go/internal/vet/vet.go | 2 + libgo/go/cmd/go/internal/work/build.go | 9 - libgo/go/cmd/go/internal/work/exec.go | 1 + libgo/go/cmd/go/internal/work/security.go | 1 + libgo/go/cmd/go/internal/work/security_test.go | 1 + libgo/go/cmd/go/internal/workcmd/edit.go | 46 ++-- libgo/go/cmd/go/internal/workcmd/init.go | 11 +- libgo/go/cmd/go/internal/workcmd/sync.go | 7 +- libgo/go/cmd/go/internal/workcmd/use.go | 147 +++++++--- libgo/go/cmd/go/internal/workcmd/work.go | 2 +- libgo/go/cmd/go/script_test.go | 8 +- libgo/go/cmd/go/testdata/script/build_internal.txt | 2 + .../go/testdata/script/mod_download_partial.txt | 11 +- .../go/cmd/go/testdata/script/mod_fs_patterns.txt | 6 +- .../cmd/go/testdata/script/mod_invalid_version.txt | 10 +- libgo/go/cmd/go/testdata/script/mod_list_dir.txt | 2 +- .../go/testdata/script/mod_list_replace_dir.txt | 2 +- libgo/go/cmd/go/testdata/script/run_issue51125.txt | 54 ++++ .../cmd/go/testdata/script/test_fuzz_dup_cache.txt | 52 ++++ .../go/cmd/go/testdata/script/test_fuzz_return.txt | 19 ++ .../go/testdata/script/test_relative_cmdline.txt | 2 + libgo/go/cmd/go/testdata/script/work.txt | 8 +- libgo/go/cmd/go/testdata/script/work_edit.txt | 3 +- libgo/go/cmd/go/testdata/script/work_env.txt | 4 + libgo/go/cmd/go/testdata/script/work_gowork.txt | 24 ++ .../go/cmd/go/testdata/script/work_init_gowork.txt | 19 ++ .../go/cmd/go/testdata/script/work_issue51204.txt | 57 ++++ .../testdata/script/work_module_not_in_go_work.txt | 25 ++ libgo/go/cmd/go/testdata/script/work_nowork.txt | 20 ++ .../go/testdata/script/work_replace_conflict.txt | 6 +- .../go/cmd/go/testdata/script/work_use_deleted.txt | 22 ++ libgo/go/cmd/go/testdata/script/work_use_dot.txt | 49 ++++ .../cmd/go/testdata/script/work_use_issue50958.txt | 17 ++ libgo/go/cmd/go/testdata/script/work_vet.txt | 19 ++ libgo/go/cmd/go/testdata/script/work_workfile.txt | 21 -- libgo/go/cmd/gofmt/gofmt.go | 128 +++++---- libgo/go/cmd/internal/objabi/funcdata.go | 1 + 58 files changed, 1027 insertions(+), 507 deletions(-) create mode 100644 libgo/go/cmd/go/testdata/script/run_issue51125.txt create mode 100644 libgo/go/cmd/go/testdata/script/test_fuzz_dup_cache.txt create mode 100644 libgo/go/cmd/go/testdata/script/test_fuzz_return.txt create mode 100644 libgo/go/cmd/go/testdata/script/work_gowork.txt create mode 100644 libgo/go/cmd/go/testdata/script/work_init_gowork.txt create mode 100644 libgo/go/cmd/go/testdata/script/work_issue51204.txt create mode 100644 libgo/go/cmd/go/testdata/script/work_module_not_in_go_work.txt create mode 100644 libgo/go/cmd/go/testdata/script/work_nowork.txt create mode 100644 libgo/go/cmd/go/testdata/script/work_use_deleted.txt create mode 100644 libgo/go/cmd/go/testdata/script/work_use_dot.txt create mode 100644 libgo/go/cmd/go/testdata/script/work_use_issue50958.txt create mode 100644 libgo/go/cmd/go/testdata/script/work_vet.txt delete mode 100644 libgo/go/cmd/go/testdata/script/work_workfile.txt (limited to 'libgo/go/cmd') diff --git a/libgo/go/cmd/go/alldocs.go b/libgo/go/cmd/go/alldocs.go index 826b0cc..63e7900 100644 --- a/libgo/go/cmd/go/alldocs.go +++ b/libgo/go/cmd/go/alldocs.go @@ -177,14 +177,6 @@ // directory, but it is not accessed. When -modfile is specified, an // alternate go.sum file is also used: its path is derived from the // -modfile flag by trimming the ".mod" extension and appending ".sum". -// -workfile file -// in module aware mode, use the given go.work file as a workspace file. -// By default or when -workfile is "auto", the go command searches for a -// file named go.work in the current directory and then containing directories -// until one is found. If a valid go.work file is found, the modules -// specified will collectively be used as the main modules. If -workfile -// is "off", or a go.work file is not found in "auto" mode, workspace -// mode is disabled. // -overlay file // read a JSON config file that provides an overlay for build operations. // The file is a JSON struct with a single field, named 'Replace', that @@ -1379,7 +1371,7 @@ // builds from local modules. // // go.work files are line-oriented. Each line holds a single directive, -// made up of a keyword followed by aruments. For example: +// made up of a keyword followed by arguments. For example: // // go 1.18 // @@ -1472,19 +1464,14 @@ // The -json flag prints the final go.work file in JSON format instead of // writing it back to go.mod. The JSON output corresponds to these Go types: // -// type Module struct { -// Path string -// Version string -// } -// // type GoWork struct { -// Go string -// Directory []Directory -// Replace []Replace +// Go string +// Use []Use +// Replace []Replace // } // // type Use struct { -// Path string +// DiskPath string // ModulePath string // } // @@ -1493,6 +1480,11 @@ // New Module // } // +// type Module struct { +// Path string +// Version string +// } +// // See the workspaces design proposal at // https://go.googlesource.com/proposal/+/master/design/45713-workspace.md for // more information. @@ -2036,6 +2028,8 @@ // GOENV // The location of the Go environment configuration file. // Cannot be set using 'go env -w'. +// Setting GOENV=off in the environment disables the use of the +// default configuration file. // GOFLAGS // A space-separated list of -flag=value settings to apply // to go commands by default, when the given flag is known by @@ -2073,6 +2067,14 @@ // GOVCS // Lists version control commands that may be used with matching servers. // See 'go help vcs'. +// GOWORK +// In module aware mode, use the given go.work file as a workspace file. +// By default or when GOWORK is "auto", the go command searches for a +// file named go.work in the current directory and then containing directories +// until one is found. If a valid go.work file is found, the modules +// specified will collectively be used as the main modules. If GOWORK +// is "off", or a go.work file is not found in "auto" mode, workspace +// mode is disabled. // // Environment variables for use with cgo: // diff --git a/libgo/go/cmd/go/go_test.go b/libgo/go/cmd/go/go_test.go index 7aaec4eb..1ea347c 100644 --- a/libgo/go/cmd/go/go_test.go +++ b/libgo/go/cmd/go/go_test.go @@ -133,7 +133,7 @@ func TestMain(m *testing.M) { } gotool, err := testenv.GoTool() if err != nil { - fmt.Fprintln(os.Stderr, err) + fmt.Fprintln(os.Stderr, "locating go tool: ", err) os.Exit(2) } diff --git a/libgo/go/cmd/go/internal/base/flag.go b/libgo/go/cmd/go/internal/base/flag.go index 2c72c7e..120420a 100644 --- a/libgo/go/cmd/go/internal/base/flag.go +++ b/libgo/go/cmd/go/internal/base/flag.go @@ -62,13 +62,6 @@ func AddModFlag(flags *flag.FlagSet) { flags.Var(explicitStringFlag{value: &cfg.BuildMod, explicit: &cfg.BuildModExplicit}, "mod", "") } -// AddWorkfileFlag adds the workfile flag to the flag set. It enables workspace -// mode for commands that support it by resetting the cfg.WorkFile variable -// to "" (equivalent to auto) rather than off. -func AddWorkfileFlag(flags *flag.FlagSet) { - flags.Var(explicitStringFlag{value: &cfg.WorkFile, explicit: &cfg.WorkFileExplicit}, "workfile", "") -} - // AddModCommonFlags adds the module-related flags common to build commands // and 'go mod' subcommands. func AddModCommonFlags(flags *flag.FlagSet) { diff --git a/libgo/go/cmd/go/internal/cfg/cfg.go b/libgo/go/cmd/go/internal/cfg/cfg.go index 7f68d7b..deab3dd 100644 --- a/libgo/go/cmd/go/internal/cfg/cfg.go +++ b/libgo/go/cmd/go/internal/cfg/cfg.go @@ -49,10 +49,8 @@ var ( BuildWork bool // -work flag BuildX bool // -x flag - ModCacheRW bool // -modcacherw flag - ModFile string // -modfile flag - WorkFile string // -workfile flag - WorkFileExplicit bool // whether -workfile was set explicitly + ModCacheRW bool // -modcacherw flag + ModFile string // -modfile flag CmdName string // "build", "install", "list", "mod tidy", etc. diff --git a/libgo/go/cmd/go/internal/envcmd/env.go b/libgo/go/cmd/go/internal/envcmd/env.go index e56dd82..c1adf8c 100644 --- a/libgo/go/cmd/go/internal/envcmd/env.go +++ b/libgo/go/cmd/go/internal/envcmd/env.go @@ -154,6 +154,10 @@ func ExtraEnvVars() []cfg.EnvVar { } modload.InitWorkfile() gowork := modload.WorkFilePath() + // As a special case, if a user set off explicitly, report that in GOWORK. + if cfg.Getenv("GOWORK") == "off" { + gowork = "off" + } return []cfg.EnvVar{ {Name: "GOMOD", Value: gomod}, {Name: "GOWORK", Value: gowork}, diff --git a/libgo/go/cmd/go/internal/help/helpdoc.go b/libgo/go/cmd/go/internal/help/helpdoc.go index 7dc066c..28ddaac 100644 --- a/libgo/go/cmd/go/internal/help/helpdoc.go +++ b/libgo/go/cmd/go/internal/help/helpdoc.go @@ -506,6 +506,8 @@ General-purpose environment variables: GOENV The location of the Go environment configuration file. Cannot be set using 'go env -w'. + Setting GOENV=off in the environment disables the use of the + default configuration file. GOFLAGS A space-separated list of -flag=value settings to apply to go commands by default, when the given flag is known by @@ -543,6 +545,14 @@ General-purpose environment variables: GOVCS Lists version control commands that may be used with matching servers. See 'go help vcs'. + GOWORK + In module aware mode, use the given go.work file as a workspace file. + By default or when GOWORK is "auto", the go command searches for a + file named go.work in the current directory and then containing directories + until one is found. If a valid go.work file is found, the modules + specified will collectively be used as the main modules. If GOWORK + is "off", or a go.work file is not found in "auto" mode, workspace + mode is disabled. Environment variables for use with cgo: diff --git a/libgo/go/cmd/go/internal/list/list.go b/libgo/go/cmd/go/internal/list/list.go index d9a7078..8be9211 100644 --- a/libgo/go/cmd/go/internal/list/list.go +++ b/libgo/go/cmd/go/internal/list/list.go @@ -316,7 +316,6 @@ For more about modules, see https://golang.org/ref/mod. func init() { CmdList.Run = runList // break init cycle work.AddBuildFlags(CmdList, work.DefaultBuildFlags) - base.AddWorkfileFlag(&CmdList.Flag) } var ( diff --git a/libgo/go/cmd/go/internal/load/pkg.go b/libgo/go/cmd/go/internal/load/pkg.go index 67b359d..44bd61c 100644 --- a/libgo/go/cmd/go/internal/load/pkg.go +++ b/libgo/go/cmd/go/internal/load/pkg.go @@ -819,11 +819,11 @@ func loadPackageData(ctx context.Context, path, parentPath, parentDir, parentRoo } r := resolvedImportCache.Do(importKey, func() any { var r resolvedImport - if build.IsLocalImport(path) { + if cfg.ModulesEnabled { + r.dir, r.path, r.err = modload.Lookup(parentPath, parentIsStd, path) + } else if build.IsLocalImport(path) { r.dir = filepath.Join(parentDir, path) r.path = dirToImportPath(r.dir) - } else if cfg.ModulesEnabled { - r.dir, r.path, r.err = modload.Lookup(parentPath, parentIsStd, path) } else if mode&ResolveImport != 0 { // We do our own path resolution, because we want to // find out the key to use in packageCache without the @@ -1113,6 +1113,7 @@ func dirAndRoot(path string, dir, root string) (string, string) { } if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || path != "command-line-arguments" && !build.IsLocalImport(path) && filepath.Join(root, path) != dir { + debug.PrintStack() base.Fatalf("unexpected directory layout:\n"+ " import path: %s\n"+ " root: %s\n"+ @@ -2235,13 +2236,17 @@ func (p *Package) setBuildInfo() { var debugModFromModinfo func(*modinfo.ModulePublic) *debug.Module debugModFromModinfo = func(mi *modinfo.ModulePublic) *debug.Module { + version := mi.Version + if version == "" { + version = "(devel)" + } dm := &debug.Module{ Path: mi.Path, - Version: mi.Version, + Version: version, } if mi.Replace != nil { dm.Replace = debugModFromModinfo(mi.Replace) - } else { + } else if mi.Version != "" { dm.Sum = modfetch.Sum(module.Version{Path: mi.Path, Version: mi.Version}) } return dm @@ -2424,12 +2429,7 @@ func (p *Package) setBuildInfo() { appendSetting("vcs.modified", strconv.FormatBool(st.Uncommitted)) } - text, err := info.MarshalText() - if err != nil { - setPkgErrorf("error formatting build info: %v", err) - return - } - p.Internal.BuildInfo = string(text) + p.Internal.BuildInfo = info.String() } // SafeArg reports whether arg is a "safe" command-line argument, diff --git a/libgo/go/cmd/go/internal/modcmd/download.go b/libgo/go/cmd/go/internal/modcmd/download.go index 6b8a010..5bc6cbc 100644 --- a/libgo/go/cmd/go/internal/modcmd/download.go +++ b/libgo/go/cmd/go/internal/modcmd/download.go @@ -70,7 +70,6 @@ func init() { // TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands. cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "") base.AddModCommonFlags(&cmdDownload.Flag) - base.AddWorkfileFlag(&cmdDownload.Flag) } type moduleJSON struct { diff --git a/libgo/go/cmd/go/internal/modcmd/graph.go b/libgo/go/cmd/go/internal/modcmd/graph.go index 9b6aa1f..9568c65 100644 --- a/libgo/go/cmd/go/internal/modcmd/graph.go +++ b/libgo/go/cmd/go/internal/modcmd/graph.go @@ -42,7 +42,6 @@ var ( func init() { cmdGraph.Flag.Var(&graphGo, "go", "") base.AddModCommonFlags(&cmdGraph.Flag) - base.AddWorkfileFlag(&cmdGraph.Flag) } func runGraph(ctx context.Context, cmd *base.Command, args []string) { diff --git a/libgo/go/cmd/go/internal/modcmd/verify.go b/libgo/go/cmd/go/internal/modcmd/verify.go index 3f0c005..459bf5d 100644 --- a/libgo/go/cmd/go/internal/modcmd/verify.go +++ b/libgo/go/cmd/go/internal/modcmd/verify.go @@ -39,7 +39,6 @@ See https://golang.org/ref/mod#go-mod-verify for more about 'go mod verify'. func init() { base.AddModCommonFlags(&cmdVerify.Flag) - base.AddWorkfileFlag(&cmdVerify.Flag) } func runVerify(ctx context.Context, cmd *base.Command, args []string) { diff --git a/libgo/go/cmd/go/internal/modcmd/why.go b/libgo/go/cmd/go/internal/modcmd/why.go index d8355cc..2d3f1eb 100644 --- a/libgo/go/cmd/go/internal/modcmd/why.go +++ b/libgo/go/cmd/go/internal/modcmd/why.go @@ -59,7 +59,6 @@ var ( func init() { cmdWhy.Run = runWhy // break init cycle base.AddModCommonFlags(&cmdWhy.Flag) - base.AddWorkfileFlag(&cmdWhy.Flag) } func runWhy(ctx context.Context, cmd *base.Command, args []string) { diff --git a/libgo/go/cmd/go/internal/modfetch/coderepo.go b/libgo/go/cmd/go/internal/modfetch/coderepo.go index 79da010..2206c7c 100644 --- a/libgo/go/cmd/go/internal/modfetch/coderepo.go +++ b/libgo/go/cmd/go/internal/modfetch/coderepo.go @@ -298,16 +298,13 @@ func (r *codeRepo) Latest() (*RevInfo, error) { // If statVers is a valid module version, it is used for the Version field. // Otherwise, the Version is derived from the passed-in info and recent tags. func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, error) { - info2 := &RevInfo{ - Name: info.Name, - Short: info.Short, - Time: info.Time, - } - // If this is a plain tag (no dir/ prefix) // and the module path is unversioned, // and if the underlying file tree has no go.mod, // then allow using the tag with a +incompatible suffix. + // + // (If the version is +incompatible, then the go.mod file must not exist: + // +incompatible is not an ongoing opt-out from semantic import versioning.) var canUseIncompatible func() bool canUseIncompatible = func() bool { var ok bool @@ -321,19 +318,12 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e return ok } - invalidf := func(format string, args ...any) error { - return &module.ModuleError{ - Path: r.modPath, - Err: &module.InvalidVersionError{ - Version: info2.Version, - Err: fmt.Errorf(format, args...), - }, - } - } - - // checkGoMod verifies that the go.mod file for the module exists or does not - // exist as required by info2.Version and the module path represented by r. - checkGoMod := func() (*RevInfo, error) { + // checkCanonical verifies that the canonical version v is compatible with the + // module path represented by r, adding a "+incompatible" suffix if needed. + // + // If statVers is also canonical, checkCanonical also verifies that v is + // either statVers or statVers with the added "+incompatible" suffix. + checkCanonical := func(v string) (*RevInfo, error) { // If r.codeDir is non-empty, then the go.mod file must exist: the module // author — not the module consumer, — gets to decide how to carve up the repo // into modules. @@ -344,73 +334,91 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e // r.findDir verifies both of these conditions. Execute it now so that // r.Stat will correctly return a notExistError if the go.mod location or // declared module path doesn't match. - _, _, _, err := r.findDir(info2.Version) + _, _, _, err := r.findDir(v) if err != nil { // TODO: It would be nice to return an error like "not a module". // Right now we return "missing go.mod", which is a little confusing. return nil, &module.ModuleError{ Path: r.modPath, Err: &module.InvalidVersionError{ - Version: info2.Version, + Version: v, Err: notExistError{err: err}, }, } } - // If the version is +incompatible, then the go.mod file must not exist: - // +incompatible is not an ongoing opt-out from semantic import versioning. - if strings.HasSuffix(info2.Version, "+incompatible") { - if !canUseIncompatible() { + invalidf := func(format string, args ...any) error { + return &module.ModuleError{ + Path: r.modPath, + Err: &module.InvalidVersionError{ + Version: v, + Err: fmt.Errorf(format, args...), + }, + } + } + + // Add the +incompatible suffix if needed or requested explicitly, and + // verify that its presence or absence is appropriate for this version + // (which depends on whether it has an explicit go.mod file). + + if v == strings.TrimSuffix(statVers, "+incompatible") { + v = statVers + } + base := strings.TrimSuffix(v, "+incompatible") + var errIncompatible error + if !module.MatchPathMajor(base, r.pathMajor) { + if canUseIncompatible() { + v = base + "+incompatible" + } else { if r.pathMajor != "" { - return nil, invalidf("+incompatible suffix not allowed: module path includes a major version suffix, so major version must match") + errIncompatible = invalidf("module path includes a major version suffix, so major version must match") } else { - return nil, invalidf("+incompatible suffix not allowed: module contains a go.mod file, so semantic import versioning is required") + errIncompatible = invalidf("module contains a go.mod file, so module path must match major version (%q)", path.Join(r.pathPrefix, semver.Major(v))) } } + } else if strings.HasSuffix(v, "+incompatible") { + errIncompatible = invalidf("+incompatible suffix not allowed: major version %s is compatible", semver.Major(v)) + } - if err := module.CheckPathMajor(strings.TrimSuffix(info2.Version, "+incompatible"), r.pathMajor); err == nil { - return nil, invalidf("+incompatible suffix not allowed: major version %s is compatible", semver.Major(info2.Version)) + if statVers != "" && statVers == module.CanonicalVersion(statVers) { + // Since the caller-requested version is canonical, it would be very + // confusing to resolve it to anything but itself, possibly with a + // "+incompatible" suffix. Error out explicitly. + if statBase := strings.TrimSuffix(statVers, "+incompatible"); statBase != base { + return nil, &module.ModuleError{ + Path: r.modPath, + Err: &module.InvalidVersionError{ + Version: statVers, + Err: fmt.Errorf("resolves to version %v (%s is not a tag)", v, statBase), + }, + } } } - return info2, nil + if errIncompatible != nil { + return nil, errIncompatible + } + + return &RevInfo{ + Name: info.Name, + Short: info.Short, + Time: info.Time, + Version: v, + }, nil } // Determine version. - // - // If statVers is canonical, then the original call was repo.Stat(statVers). - // Since the version is canonical, we must not resolve it to anything but - // itself, possibly with a '+incompatible' annotation: we do not need to do - // the work required to look for an arbitrary pseudo-version. - if statVers != "" && statVers == module.CanonicalVersion(statVers) { - info2.Version = statVers - - if module.IsPseudoVersion(info2.Version) { - if err := r.validatePseudoVersion(info, info2.Version); err != nil { - return nil, err - } - return checkGoMod() - } - if err := module.CheckPathMajor(info2.Version, r.pathMajor); err != nil { - if canUseIncompatible() { - info2.Version += "+incompatible" - return checkGoMod() - } else { - if vErr, ok := err.(*module.InvalidVersionError); ok { - // We're going to describe why the version is invalid in more detail, - // so strip out the existing “invalid version” wrapper. - err = vErr.Err - } - return nil, invalidf("module contains a go.mod file, so major version must be compatible: %v", err) - } + if module.IsPseudoVersion(statVers) { + if err := r.validatePseudoVersion(info, statVers); err != nil { + return nil, err } - - return checkGoMod() + return checkCanonical(statVers) } - // statVers is empty or non-canonical, so we need to resolve it to a canonical - // version or pseudo-version. + // statVers is not a pseudo-version, so we need to either resolve it to a + // canonical version or verify that it is already a canonical tag + // (not a branch). // Derive or verify a version from a code repo tag. // Tag must have a prefix matching codeDir. @@ -441,71 +449,62 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e if v == "" || !strings.HasPrefix(trimmed, v) { return "", false // Invalid or incomplete version (just vX or vX.Y). } - if isRetracted(v) { - return "", false - } if v == trimmed { tagIsCanonical = true } - - if err := module.CheckPathMajor(v, r.pathMajor); err != nil { - if canUseIncompatible() { - return v + "+incompatible", tagIsCanonical - } - return "", false - } - return v, tagIsCanonical } // If the VCS gave us a valid version, use that. if v, tagIsCanonical := tagToVersion(info.Version); tagIsCanonical { - info2.Version = v - return checkGoMod() + if info, err := checkCanonical(v); err == nil { + return info, err + } } // Look through the tags on the revision for either a usable canonical version // or an appropriate base for a pseudo-version. - var pseudoBase string + var ( + highestCanonical string + pseudoBase string + ) for _, pathTag := range info.Tags { v, tagIsCanonical := tagToVersion(pathTag) - if tagIsCanonical { - if statVers != "" && semver.Compare(v, statVers) == 0 { - // The user requested a non-canonical version, but the tag for the - // canonical equivalent refers to the same revision. Use it. - info2.Version = v - return checkGoMod() + if statVers != "" && semver.Compare(v, statVers) == 0 { + // The tag is equivalent to the version requested by the user. + if tagIsCanonical { + // This tag is the canonical form of the requested version, + // not some other form with extra build metadata. + // Use this tag so that the resolved version will match exactly. + // (If it isn't actually allowed, we'll error out in checkCanonical.) + return checkCanonical(v) } else { - // Save the highest canonical tag for the revision. If we don't find a - // better match, we'll use it as the canonical version. + // The user explicitly requested something equivalent to this tag. We + // can't use the version from the tag directly: since the tag is not + // canonical, it could be ambiguous. For example, tags v0.0.1+a and + // v0.0.1+b might both exist and refer to different revisions. // - // NOTE: Do not replace this with semver.Max. Despite the name, - // semver.Max *also* canonicalizes its arguments, which uses - // semver.Canonical instead of module.CanonicalVersion and thereby - // strips our "+incompatible" suffix. - if semver.Compare(info2.Version, v) < 0 { - info2.Version = v - } + // The tag is otherwise valid for the module, so we can at least use it as + // the base of an unambiguous pseudo-version. + // + // If multiple tags match, tagToVersion will canonicalize them to the same + // base version. + pseudoBase = v + } + } + // Save the highest non-retracted canonical tag for the revision. + // If we don't find a better match, we'll use it as the canonical version. + if tagIsCanonical && semver.Compare(highestCanonical, v) < 0 && !isRetracted(v) { + if module.MatchPathMajor(v, r.pathMajor) || canUseIncompatible() { + highestCanonical = v } - } else if v != "" && semver.Compare(v, statVers) == 0 { - // The user explicitly requested something equivalent to this tag. We - // can't use the version from the tag directly: since the tag is not - // canonical, it could be ambiguous. For example, tags v0.0.1+a and - // v0.0.1+b might both exist and refer to different revisions. - // - // The tag is otherwise valid for the module, so we can at least use it as - // the base of an unambiguous pseudo-version. - // - // If multiple tags match, tagToVersion will canonicalize them to the same - // base version. - pseudoBase = v } } - // If we found any canonical tag for the revision, return it. + // If we found a valid canonical tag for the revision, return it. // Even if we found a good pseudo-version base, a canonical version is better. - if info2.Version != "" { - return checkGoMod() + if highestCanonical != "" { + return checkCanonical(highestCanonical) } // Find the highest tagged version in the revision's history, subject to @@ -528,11 +527,10 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v0")) } } - pseudoBase, _ = tagToVersion(tag) // empty if the tag is invalid + pseudoBase, _ = tagToVersion(tag) } - info2.Version = module.PseudoVersion(r.pseudoMajor, pseudoBase, info.Time, info.Short) - return checkGoMod() + return checkCanonical(module.PseudoVersion(r.pseudoMajor, pseudoBase, info.Time, info.Short)) } // validatePseudoVersion checks that version has a major version compatible with @@ -556,10 +554,6 @@ func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string) } }() - if err := module.CheckPathMajor(version, r.pathMajor); err != nil { - return err - } - rev, err := module.PseudoVersionRev(version) if err != nil { return err diff --git a/libgo/go/cmd/go/internal/modfetch/coderepo_test.go b/libgo/go/cmd/go/internal/modfetch/coderepo_test.go index 02e399f..d98ea87 100644 --- a/libgo/go/cmd/go/internal/modfetch/coderepo_test.go +++ b/libgo/go/cmd/go/internal/modfetch/coderepo_test.go @@ -418,171 +418,204 @@ var codeRepoTests = []codeRepoTest{ zipSum: "h1:JItBZ+gwA5WvtZEGEbuDL4lUttGtLrs53lmdurq3bOg=", zipFileHash: "9ea9ae1673cffcc44b7fdd3cc89953d68c102449b46c982dbf085e4f2e394da5", }, + { + // Git branch with a semver name, +incompatible version, and no go.mod file. + vcs: "git", + path: "vcs-test.golang.org/go/mod/gitrepo1", + rev: "v2.3.4+incompatible", + err: `resolves to version v2.0.1+incompatible (v2.3.4 is not a tag)`, + }, + { + // Git branch with a semver name, matching go.mod file, and compatible version. + vcs: "git", + path: "vcs-test.golang.org/git/semver-branch.git", + rev: "v1.0.0", + err: `resolves to version v0.1.1-0.20220202191944-09c4d8f6938c (v1.0.0 is not a tag)`, + }, + { + // Git branch with a semver name, matching go.mod file, and disallowed +incompatible version. + // The version/tag mismatch takes precedence over the +incompatible mismatched. + vcs: "git", + path: "vcs-test.golang.org/git/semver-branch.git", + rev: "v2.0.0+incompatible", + err: `resolves to version v0.1.0 (v2.0.0 is not a tag)`, + }, + { + // Git branch with a semver name, matching go.mod file, and mismatched version. + // The version/tag mismatch takes precedence over the +incompatible mismatched. + vcs: "git", + path: "vcs-test.golang.org/git/semver-branch.git", + rev: "v2.0.0", + err: `resolves to version v0.1.0 (v2.0.0 is not a tag)`, + }, + { + // v3.0.0-devel is the same as tag v4.0.0-beta.1, but v4.0.0-beta.1 would + // not be allowed because it is incompatible and a go.mod file exists. + // The error message should refer to a valid pseudo-version, not the + // unusable semver tag. + vcs: "git", + path: "vcs-test.golang.org/git/semver-branch.git", + rev: "v3.0.0-devel", + err: `resolves to version v0.1.1-0.20220203155313-d59622f6e4d7 (v3.0.0-devel is not a tag)`, + }, } func TestCodeRepo(t *testing.T) { testenv.MustHaveExternalNetwork(t) + tmpdir := t.TempDir() - tmpdir, err := os.MkdirTemp("", "modfetch-test-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) + for _, tt := range codeRepoTests { + f := func(tt codeRepoTest) func(t *testing.T) { + return func(t *testing.T) { + t.Parallel() + if tt.vcs != "mod" { + testenv.MustHaveExecPath(t, tt.vcs) + } - t.Run("parallel", func(t *testing.T) { - for _, tt := range codeRepoTests { - f := func(tt codeRepoTest) func(t *testing.T) { - return func(t *testing.T) { - t.Parallel() - if tt.vcs != "mod" { - testenv.MustHaveExecPath(t, tt.vcs) - } + repo := Lookup("direct", tt.path) - repo := Lookup("direct", tt.path) + if tt.mpath == "" { + tt.mpath = tt.path + } + if mpath := repo.ModulePath(); mpath != tt.mpath { + t.Errorf("repo.ModulePath() = %q, want %q", mpath, tt.mpath) + } - if tt.mpath == "" { - tt.mpath = tt.path - } - if mpath := repo.ModulePath(); mpath != tt.mpath { - t.Errorf("repo.ModulePath() = %q, want %q", mpath, tt.mpath) + info, err := repo.Stat(tt.rev) + if err != nil { + if tt.err != "" { + if !strings.Contains(err.Error(), tt.err) { + t.Fatalf("repoStat(%q): %v, wanted %q", tt.rev, err, tt.err) + } + return } + t.Fatalf("repo.Stat(%q): %v", tt.rev, err) + } + if tt.err != "" { + t.Errorf("repo.Stat(%q): success, wanted error", tt.rev) + } + if info.Version != tt.version { + t.Errorf("info.Version = %q, want %q", info.Version, tt.version) + } + if info.Name != tt.name { + t.Errorf("info.Name = %q, want %q", info.Name, tt.name) + } + if info.Short != tt.short { + t.Errorf("info.Short = %q, want %q", info.Short, tt.short) + } + if !info.Time.Equal(tt.time) { + t.Errorf("info.Time = %v, want %v", info.Time, tt.time) + } - info, err := repo.Stat(tt.rev) - if err != nil { - if tt.err != "" { - if !strings.Contains(err.Error(), tt.err) { - t.Fatalf("repoStat(%q): %v, wanted %q", tt.rev, err, tt.err) - } - return + if tt.gomod != "" || tt.gomodErr != "" { + data, err := repo.GoMod(tt.version) + if err != nil && tt.gomodErr == "" { + t.Errorf("repo.GoMod(%q): %v", tt.version, err) + } else if err != nil && tt.gomodErr != "" { + if err.Error() != tt.gomodErr { + t.Errorf("repo.GoMod(%q): %v, want %q", tt.version, err, tt.gomodErr) } - t.Fatalf("repo.Stat(%q): %v", tt.rev, err) - } - if tt.err != "" { - t.Errorf("repo.Stat(%q): success, wanted error", tt.rev) - } - if info.Version != tt.version { - t.Errorf("info.Version = %q, want %q", info.Version, tt.version) + } else if tt.gomodErr != "" { + t.Errorf("repo.GoMod(%q) = %q, want error %q", tt.version, data, tt.gomodErr) + } else if string(data) != tt.gomod { + t.Errorf("repo.GoMod(%q) = %q, want %q", tt.version, data, tt.gomod) } - if info.Name != tt.name { - t.Errorf("info.Name = %q, want %q", info.Name, tt.name) - } - if info.Short != tt.short { - t.Errorf("info.Short = %q, want %q", info.Short, tt.short) + } + + needHash := !testing.Short() && (tt.zipFileHash != "" || tt.zipSum != "") + if tt.zip != nil || tt.zipErr != "" || needHash { + f, err := os.CreateTemp(tmpdir, tt.version+".zip.") + if err != nil { + t.Fatalf("os.CreateTemp: %v", err) } - if !info.Time.Equal(tt.time) { - t.Errorf("info.Time = %v, want %v", info.Time, tt.time) + zipfile := f.Name() + defer func() { + f.Close() + os.Remove(zipfile) + }() + + var w io.Writer + var h hash.Hash + if needHash { + h = sha256.New() + w = io.MultiWriter(f, h) + } else { + w = f } - - if tt.gomod != "" || tt.gomodErr != "" { - data, err := repo.GoMod(tt.version) - if err != nil && tt.gomodErr == "" { - t.Errorf("repo.GoMod(%q): %v", tt.version, err) - } else if err != nil && tt.gomodErr != "" { - if err.Error() != tt.gomodErr { - t.Errorf("repo.GoMod(%q): %v, want %q", tt.version, err, tt.gomodErr) + err = repo.Zip(w, tt.version) + f.Close() + if err != nil { + if tt.zipErr != "" { + if err.Error() == tt.zipErr { + return } - } else if tt.gomodErr != "" { - t.Errorf("repo.GoMod(%q) = %q, want error %q", tt.version, data, tt.gomodErr) - } else if string(data) != tt.gomod { - t.Errorf("repo.GoMod(%q) = %q, want %q", tt.version, data, tt.gomod) + t.Fatalf("repo.Zip(%q): %v, want error %q", tt.version, err, tt.zipErr) } + t.Fatalf("repo.Zip(%q): %v", tt.version, err) + } + if tt.zipErr != "" { + t.Errorf("repo.Zip(%q): success, want error %q", tt.version, tt.zipErr) } - needHash := !testing.Short() && (tt.zipFileHash != "" || tt.zipSum != "") - if tt.zip != nil || tt.zipErr != "" || needHash { - f, err := os.CreateTemp(tmpdir, tt.version+".zip.") + if tt.zip != nil { + prefix := tt.path + "@" + tt.version + "/" + z, err := zip.OpenReader(zipfile) if err != nil { - t.Fatalf("os.CreateTemp: %v", err) + t.Fatalf("open zip %s: %v", zipfile, err) } - zipfile := f.Name() - defer func() { - f.Close() - os.Remove(zipfile) - }() - - var w io.Writer - var h hash.Hash - if needHash { - h = sha256.New() - w = io.MultiWriter(f, h) - } else { - w = f - } - err = repo.Zip(w, tt.version) - f.Close() - if err != nil { - if tt.zipErr != "" { - if err.Error() == tt.zipErr { - return - } - t.Fatalf("repo.Zip(%q): %v, want error %q", tt.version, err, tt.zipErr) + var names []string + for _, file := range z.File { + if !strings.HasPrefix(file.Name, prefix) { + t.Errorf("zip entry %v does not start with prefix %v", file.Name, prefix) + continue } - t.Fatalf("repo.Zip(%q): %v", tt.version, err) - } - if tt.zipErr != "" { - t.Errorf("repo.Zip(%q): success, want error %q", tt.version, tt.zipErr) + names = append(names, file.Name[len(prefix):]) } - - if tt.zip != nil { - prefix := tt.path + "@" + tt.version + "/" - z, err := zip.OpenReader(zipfile) - if err != nil { - t.Fatalf("open zip %s: %v", zipfile, err) - } - var names []string - for _, file := range z.File { - if !strings.HasPrefix(file.Name, prefix) { - t.Errorf("zip entry %v does not start with prefix %v", file.Name, prefix) - continue - } - names = append(names, file.Name[len(prefix):]) - } - z.Close() - if !reflect.DeepEqual(names, tt.zip) { - t.Fatalf("zip = %v\nwant %v\n", names, tt.zip) - } + z.Close() + if !reflect.DeepEqual(names, tt.zip) { + t.Fatalf("zip = %v\nwant %v\n", names, tt.zip) } + } - if needHash { - sum, err := dirhash.HashZip(zipfile, dirhash.Hash1) - if err != nil { - t.Errorf("repo.Zip(%q): %v", tt.version, err) - } else if sum != tt.zipSum { - t.Errorf("repo.Zip(%q): got file with sum %q, want %q", tt.version, sum, tt.zipSum) - } else if zipFileHash := hex.EncodeToString(h.Sum(nil)); zipFileHash != tt.zipFileHash { - t.Errorf("repo.Zip(%q): got file with hash %q, want %q (but content has correct sum)", tt.version, zipFileHash, tt.zipFileHash) - } + if needHash { + sum, err := dirhash.HashZip(zipfile, dirhash.Hash1) + if err != nil { + t.Errorf("repo.Zip(%q): %v", tt.version, err) + } else if sum != tt.zipSum { + t.Errorf("repo.Zip(%q): got file with sum %q, want %q", tt.version, sum, tt.zipSum) + } else if zipFileHash := hex.EncodeToString(h.Sum(nil)); zipFileHash != tt.zipFileHash { + t.Errorf("repo.Zip(%q): got file with hash %q, want %q (but content has correct sum)", tt.version, zipFileHash, tt.zipFileHash) } } } } - t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.rev, f(tt)) - if strings.HasPrefix(tt.path, vgotest1git) { - for vcs, alt := range altVgotests { - altTest := tt - altTest.vcs = vcs - altTest.path = alt + strings.TrimPrefix(altTest.path, vgotest1git) - if strings.HasPrefix(altTest.mpath, vgotest1git) { - altTest.mpath = alt + strings.TrimPrefix(altTest.mpath, vgotest1git) - } - var m map[string]string - if alt == vgotest1hg { - m = hgmap - } - altTest.version = remap(altTest.version, m) - altTest.name = remap(altTest.name, m) - altTest.short = remap(altTest.short, m) - altTest.rev = remap(altTest.rev, m) - altTest.err = remap(altTest.err, m) - altTest.gomodErr = remap(altTest.gomodErr, m) - altTest.zipErr = remap(altTest.zipErr, m) - altTest.zipSum = "" - altTest.zipFileHash = "" - t.Run(strings.ReplaceAll(altTest.path, "/", "_")+"/"+altTest.rev, f(altTest)) + } + t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.rev, f(tt)) + if strings.HasPrefix(tt.path, vgotest1git) { + for vcs, alt := range altVgotests { + altTest := tt + altTest.vcs = vcs + altTest.path = alt + strings.TrimPrefix(altTest.path, vgotest1git) + if strings.HasPrefix(altTest.mpath, vgotest1git) { + altTest.mpath = alt + strings.TrimPrefix(altTest.mpath, vgotest1git) + } + var m map[string]string + if alt == vgotest1hg { + m = hgmap } + altTest.version = remap(altTest.version, m) + altTest.name = remap(altTest.name, m) + altTest.short = remap(altTest.short, m) + altTest.rev = remap(altTest.rev, m) + altTest.err = remap(altTest.err, m) + altTest.gomodErr = remap(altTest.gomodErr, m) + altTest.zipErr = remap(altTest.zipErr, m) + altTest.zipSum = "" + altTest.zipFileHash = "" + t.Run(strings.ReplaceAll(altTest.path, "/", "_")+"/"+altTest.rev, f(altTest)) } } - }) + } } var hgmap = map[string]string{ diff --git a/libgo/go/cmd/go/internal/modfetch/fetch.go b/libgo/go/cmd/go/internal/modfetch/fetch.go index f5423b4..21d5f54 100644 --- a/libgo/go/cmd/go/internal/modfetch/fetch.go +++ b/libgo/go/cmd/go/internal/modfetch/fetch.go @@ -319,7 +319,7 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e // // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns // an error and does not write ziphashfile. -func hashZip(mod module.Version, zipfile, ziphashfile string) error { +func hashZip(mod module.Version, zipfile, ziphashfile string) (err error) { hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash) if err != nil { return err @@ -331,16 +331,17 @@ func hashZip(mod module.Version, zipfile, ziphashfile string) error { if err != nil { return err } + defer func() { + if closeErr := hf.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() if err := hf.Truncate(int64(len(hash))); err != nil { return err } if _, err := hf.WriteAt([]byte(hash), 0); err != nil { return err } - if err := hf.Close(); err != nil { - return err - } - return nil } diff --git a/libgo/go/cmd/go/internal/modload/import.go b/libgo/go/cmd/go/internal/modload/import.go index 812e48a..4862f62 100644 --- a/libgo/go/cmd/go/internal/modload/import.go +++ b/libgo/go/cmd/go/internal/modload/import.go @@ -248,12 +248,26 @@ func (e *invalidImportError) Unwrap() error { // return the module, its root directory, and a list of other modules that // lexically could have provided the package but did not. func importFromModules(ctx context.Context, path string, rs *Requirements, mg *ModuleGraph) (m module.Version, dir string, altMods []module.Version, err error) { + invalidf := func(format string, args ...interface{}) (module.Version, string, []module.Version, error) { + return module.Version{}, "", nil, &invalidImportError{ + importPath: path, + err: fmt.Errorf(format, args...), + } + } + if strings.Contains(path, "@") { - return module.Version{}, "", nil, fmt.Errorf("import path should not have @version") + return invalidf("import path %q should not have @version", path) } if build.IsLocalImport(path) { - return module.Version{}, "", nil, fmt.Errorf("relative import not supported") + return invalidf("%q is relative, but relative import paths are not supported in module mode", path) } + if filepath.IsAbs(path) { + return invalidf("%q is not a package path; see 'go help packages'", path) + } + if search.IsMetaPackage(path) { + return invalidf("%q is not an importable package; see 'go help packages'", path) + } + if path == "C" { // There's no directory for import "C". return module.Version{}, "", nil, nil diff --git a/libgo/go/cmd/go/internal/modload/init.go b/libgo/go/cmd/go/internal/modload/init.go index cdcfbeb..a070666 100644 --- a/libgo/go/cmd/go/internal/modload/init.go +++ b/libgo/go/cmd/go/internal/modload/init.go @@ -288,20 +288,20 @@ func BinDir() string { // operate in workspace mode. It should not be called by other commands, // for example 'go mod tidy', that don't operate in workspace mode. func InitWorkfile() { - switch cfg.WorkFile { + switch gowork := cfg.Getenv("GOWORK"); gowork { case "off": workFilePath = "" case "", "auto": workFilePath = findWorkspaceFile(base.Cwd()) default: - if !filepath.IsAbs(cfg.WorkFile) { - base.Fatalf("the path provided to -workfile must be an absolute path") + if !filepath.IsAbs(gowork) { + base.Fatalf("the path provided to GOWORK must be an absolute path") } - workFilePath = cfg.WorkFile + workFilePath = gowork } } -// WorkFilePath returns the path of the go.work file, or "" if not in +// WorkFilePath returns the absolute path of the go.work file, or "" if not in // workspace mode. WorkFilePath must be called after InitWorkfile. func WorkFilePath() string { return workFilePath @@ -610,6 +610,9 @@ func UpdateWorkFile(wf *modfile.WorkFile) { missingModulePaths := map[string]string{} // module directory listed in file -> abspath modroot for _, d := range wf.Use { + if d.Path == "" { + continue // d is marked for deletion. + } modRoot := d.Path if d.ModulePath == "" { missingModulePaths[d.Path] = modRoot @@ -1030,11 +1033,25 @@ func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile for _, r := range modFiles[i].Replace { if replacedByWorkFile[r.Old.Path] { continue - } else if prev, ok := replacements[r.Old]; ok && !curModuleReplaces[r.Old] && prev != r.New { - base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v\nuse \"go work edit -replace %v=[override]\" to resolve", r.Old, prev, r.New, r.Old) + } + var newV module.Version = r.New + if WorkFilePath() != "" && newV.Version == "" && !filepath.IsAbs(newV.Path) { + // Since we are in a workspace, we may be loading replacements from + // multiple go.mod files. Relative paths in those replacement are + // relative to the go.mod file, not the workspace, so the same string + // may refer to two different paths and different strings may refer to + // the same path. Convert them all to be absolute instead. + // + // (We could do this outside of a workspace too, but it would mean that + // replacement paths in error strings needlessly differ from what's in + // the go.mod file.) + newV.Path = filepath.Join(rootDirs[i], newV.Path) + } + if prev, ok := replacements[r.Old]; ok && !curModuleReplaces[r.Old] && prev != newV { + base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v\nuse \"go work edit -replace %v=[override]\" to resolve", r.Old, prev, newV, r.Old) } curModuleReplaces[r.Old] = true - replacements[r.Old] = r.New + replacements[r.Old] = newV v, ok := mainModules.highestReplaced[r.Old.Path] if !ok || semver.Compare(r.Old.Version, v) > 0 { @@ -1092,7 +1109,7 @@ func setDefaultBuildMod() { if inWorkspaceMode() && cfg.BuildMod != "readonly" { base.Fatalf("go: -mod may only be set to readonly when in workspace mode, but it is set to %q"+ "\n\tRemove the -mod flag to use the default readonly value,"+ - "\n\tor set -workfile=off to disable workspace mode.", cfg.BuildMod) + "\n\tor set GOWORK=off to disable workspace mode.", cfg.BuildMod) } // Don't override an explicit '-mod=' argument. return diff --git a/libgo/go/cmd/go/internal/modload/load.go b/libgo/go/cmd/go/internal/modload/load.go index 617b634..d4847efb 100644 --- a/libgo/go/cmd/go/internal/modload/load.go +++ b/libgo/go/cmd/go/internal/modload/load.go @@ -479,7 +479,11 @@ func matchLocalDirs(ctx context.Context, modRoots []string, m *search.Match, rs } if !found && search.InDir(absDir, cfg.GOROOTsrc) == "" && pathInModuleCache(ctx, absDir, rs) == "" { m.Dirs = []string{} - m.AddError(fmt.Errorf("directory prefix %s outside available modules", base.ShortPath(absDir))) + scope := "main module or its selected dependencies" + if inWorkspaceMode() { + scope = "modules listed in go.work or their selected dependencies" + } + m.AddError(fmt.Errorf("directory prefix %s does not contain %s", base.ShortPath(absDir), scope)) return } } @@ -601,7 +605,11 @@ func resolveLocalPackage(ctx context.Context, dir string, rs *Requirements) (str pkg := pathInModuleCache(ctx, absDir, rs) if pkg == "" { - return "", fmt.Errorf("directory %s outside available modules", base.ShortPath(absDir)) + scope := "main module or its selected dependencies" + if inWorkspaceMode() { + scope = "modules listed in go.work or their selected dependencies" + } + return "", fmt.Errorf("directory %s outside %s", base.ShortPath(absDir), scope) } return pkg, nil } @@ -1667,24 +1675,6 @@ func (ld *loader) preloadRootModules(ctx context.Context, rootPkgs []string) (ch // load loads an individual package. func (ld *loader) load(ctx context.Context, pkg *loadPkg) { - if strings.Contains(pkg.path, "@") { - // Leave for error during load. - return - } - if build.IsLocalImport(pkg.path) || filepath.IsAbs(pkg.path) { - // Leave for error during load. - // (Module mode does not allow local imports.) - return - } - - if search.IsMetaPackage(pkg.path) { - pkg.err = &invalidImportError{ - importPath: pkg.path, - err: fmt.Errorf("%q is not an importable package; see 'go help packages'", pkg.path), - } - return - } - var mg *ModuleGraph if ld.requirements.pruning == unpruned { var err error diff --git a/libgo/go/cmd/go/internal/run/run.go b/libgo/go/cmd/go/internal/run/run.go index c4b70b64..00a3e4b 100644 --- a/libgo/go/cmd/go/internal/run/run.go +++ b/libgo/go/cmd/go/internal/run/run.go @@ -65,7 +65,6 @@ func init() { CmdRun.Run = runRun // break init loop work.AddBuildFlags(CmdRun, work.DefaultBuildFlags) - base.AddWorkfileFlag(&CmdRun.Flag) CmdRun.Flag.Var((*base.StringsFlag)(&work.ExecCmd), "exec", "") } diff --git a/libgo/go/cmd/go/internal/test/testflag.go b/libgo/go/cmd/go/internal/test/testflag.go index b9d1ec9..c046cac 100644 --- a/libgo/go/cmd/go/internal/test/testflag.go +++ b/libgo/go/cmd/go/internal/test/testflag.go @@ -28,7 +28,6 @@ import ( func init() { work.AddBuildFlags(CmdTest, work.OmitVFlag) - base.AddWorkfileFlag(&CmdTest.Flag) cf := CmdTest.Flag cf.BoolVar(&testC, "c", false, "") diff --git a/libgo/go/cmd/go/internal/version/version.go b/libgo/go/cmd/go/internal/version/version.go index 52502e9..1c0eb54 100644 --- a/libgo/go/cmd/go/internal/version/version.go +++ b/libgo/go/cmd/go/internal/version/version.go @@ -6,7 +6,6 @@ package version import ( - "bytes" "context" "debug/buildinfo" "errors" @@ -156,12 +155,8 @@ func scanFile(file string, info fs.FileInfo, mustPrint bool) { fmt.Printf("%s: %s\n", file, bi.GoVersion) bi.GoVersion = "" // suppress printing go version again - mod, err := bi.MarshalText() - if err != nil { - fmt.Fprintf(os.Stderr, "%s: formatting build info: %v\n", file, err) - return - } + mod := bi.String() if *versionM && len(mod) > 0 { - fmt.Printf("\t%s\n", bytes.ReplaceAll(mod[:len(mod)-1], []byte("\n"), []byte("\n\t"))) + fmt.Printf("\t%s\n", strings.ReplaceAll(mod[:len(mod)-1], "\n", "\n\t")) } } diff --git a/libgo/go/cmd/go/internal/vet/vet.go b/libgo/go/cmd/go/internal/vet/vet.go index 88b3c57..d3e0dd8 100644 --- a/libgo/go/cmd/go/internal/vet/vet.go +++ b/libgo/go/cmd/go/internal/vet/vet.go @@ -13,6 +13,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/modload" "cmd/go/internal/trace" "cmd/go/internal/work" ) @@ -54,6 +55,7 @@ See also: go fmt, go fix. func runVet(ctx context.Context, cmd *base.Command, args []string) { vetFlags, pkgArgs := vetFlags(args) + modload.InitWorkfile() // The vet command does custom flag processing; initialize workspaces after that. if cfg.DebugTrace != "" { var close func() error diff --git a/libgo/go/cmd/go/internal/work/build.go b/libgo/go/cmd/go/internal/work/build.go index 1c278d3..0b5848a 100644 --- a/libgo/go/cmd/go/internal/work/build.go +++ b/libgo/go/cmd/go/internal/work/build.go @@ -130,14 +130,6 @@ and test commands: directory, but it is not accessed. When -modfile is specified, an alternate go.sum file is also used: its path is derived from the -modfile flag by trimming the ".mod" extension and appending ".sum". - -workfile file - in module aware mode, use the given go.work file as a workspace file. - By default or when -workfile is "auto", the go command searches for a - file named go.work in the current directory and then containing directories - until one is found. If a valid go.work file is found, the modules - specified will collectively be used as the main modules. If -workfile - is "off", or a go.work file is not found in "auto" mode, workspace - mode is disabled. -overlay file read a JSON config file that provides an overlay for build operations. The file is a JSON struct with a single field, named 'Replace', that @@ -217,7 +209,6 @@ func init() { AddBuildFlags(CmdBuild, DefaultBuildFlags) AddBuildFlags(CmdInstall, DefaultBuildFlags) - base.AddWorkfileFlag(&CmdBuild.Flag) } // Note that flags consulted by other parts of the code diff --git a/libgo/go/cmd/go/internal/work/exec.go b/libgo/go/cmd/go/internal/work/exec.go index d3f0eca..a4ab060 100644 --- a/libgo/go/cmd/go/internal/work/exec.go +++ b/libgo/go/cmd/go/internal/work/exec.go @@ -2015,6 +2015,7 @@ func (b *Builder) showOutput(a *Action, dir, desc, out string) { if reldir := base.ShortPath(dir); reldir != dir { suffix = strings.ReplaceAll(suffix, " "+dir, " "+reldir) suffix = strings.ReplaceAll(suffix, "\n"+dir, "\n"+reldir) + suffix = strings.ReplaceAll(suffix, "\n\t"+dir, "\n\t"+reldir) } suffix = strings.ReplaceAll(suffix, " "+b.WorkDir, " $WORK") diff --git a/libgo/go/cmd/go/internal/work/security.go b/libgo/go/cmd/go/internal/work/security.go index e9b9f6c..d1e2c67 100644 --- a/libgo/go/cmd/go/internal/work/security.go +++ b/libgo/go/cmd/go/internal/work/security.go @@ -131,6 +131,7 @@ var validCompilerFlagsWithNextArg = []string{ "-D", "-U", "-I", + "-F", "-framework", "-include", "-isysroot", diff --git a/libgo/go/cmd/go/internal/work/security_test.go b/libgo/go/cmd/go/internal/work/security_test.go index 8d4be0a..d2aeb54 100644 --- a/libgo/go/cmd/go/internal/work/security_test.go +++ b/libgo/go/cmd/go/internal/work/security_test.go @@ -15,6 +15,7 @@ var goodCompilerFlags = [][]string{ {"-Ufoo"}, {"-Ufoo1"}, {"-F/Qt"}, + {"-F", "/Qt"}, {"-I/"}, {"-I/etc/passwd"}, {"-I."}, diff --git a/libgo/go/cmd/go/internal/workcmd/edit.go b/libgo/go/cmd/go/internal/workcmd/edit.go index c420007..05f4f3d 100644 --- a/libgo/go/cmd/go/internal/workcmd/edit.go +++ b/libgo/go/cmd/go/internal/workcmd/edit.go @@ -63,19 +63,14 @@ writing it back to go.mod. The -json flag prints the final go.work file in JSON format instead of writing it back to go.mod. The JSON output corresponds to these Go types: - type Module struct { - Path string - Version string - } - type GoWork struct { - Go string - Directory []Directory - Replace []Replace + Go string + Use []Use + Replace []Replace } type Use struct { - Path string + DiskPath string ModulePath string } @@ -84,6 +79,11 @@ writing it back to go.mod. The JSON output corresponds to these Go types: New Module } + type Module struct { + Path string + Version string + } + See the workspaces design proposal at https://go.googlesource.com/proposal/+/master/design/45713-workspace.md for more information. @@ -110,22 +110,9 @@ func init() { cmdEdit.Flag.Var(flagFunc(flagEditworkDropUse), "dropuse", "") cmdEdit.Flag.Var(flagFunc(flagEditworkReplace), "replace", "") cmdEdit.Flag.Var(flagFunc(flagEditworkDropReplace), "dropreplace", "") - - base.AddWorkfileFlag(&cmdEdit.Flag) } func runEditwork(ctx context.Context, cmd *base.Command, args []string) { - anyFlags := - *editGo != "" || - *editJSON || - *editPrint || - *editFmt || - len(workedits) > 0 - - if !anyFlags { - base.Fatalf("go: no flags specified (see 'go help work edit').") - } - if *editJSON && *editPrint { base.Fatalf("go: cannot use both -json and -print") } @@ -147,6 +134,21 @@ func runEditwork(ctx context.Context, cmd *base.Command, args []string) { } } + if gowork == "" { + base.Fatalf("go: no go.work file found\n\t(run 'go work init' first or specify path using GOWORK environment variable)") + } + + anyFlags := + *editGo != "" || + *editJSON || + *editPrint || + *editFmt || + len(workedits) > 0 + + if !anyFlags { + base.Fatalf("go: no flags specified (see 'go help work edit').") + } + workFile, err := modload.ReadWorkFile(gowork) if err != nil { base.Fatalf("go: errors parsing %s:\n%s", base.ShortPath(gowork), err) diff --git a/libgo/go/cmd/go/internal/workcmd/init.go b/libgo/go/cmd/go/internal/workcmd/init.go index cefecee..63bee6e 100644 --- a/libgo/go/cmd/go/internal/workcmd/init.go +++ b/libgo/go/cmd/go/internal/workcmd/init.go @@ -33,7 +33,6 @@ current go version will also be listed in the go.work file. func init() { base.AddModCommonFlags(&cmdInit.Flag) - base.AddWorkfileFlag(&cmdInit.Flag) } func runInit(ctx context.Context, cmd *base.Command, args []string) { @@ -41,12 +40,10 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { modload.ForceUseModules = true - // TODO(matloob): support using the -workfile path - // To do that properly, we'll have to make the module directories - // make dirs relative to workFile path before adding the paths to - // the directory entries - - workFile := filepath.Join(base.Cwd(), "go.work") + workFile := modload.WorkFilePath() + if workFile == "" { + workFile = filepath.Join(base.Cwd(), "go.work") + } modload.CreateWorkFile(ctx, workFile, args) } diff --git a/libgo/go/cmd/go/internal/workcmd/sync.go b/libgo/go/cmd/go/internal/workcmd/sync.go index 1cca817..b0f61c5 100644 --- a/libgo/go/cmd/go/internal/workcmd/sync.go +++ b/libgo/go/cmd/go/internal/workcmd/sync.go @@ -39,13 +39,14 @@ that in each workspace module. func init() { base.AddModCommonFlags(&cmdSync.Flag) - base.AddWorkfileFlag(&cmdSync.Flag) } func runSync(ctx context.Context, cmd *base.Command, args []string) { - modload.InitWorkfile() - modload.ForceUseModules = true + modload.InitWorkfile() + if modload.WorkFilePath() == "" { + base.Fatalf("go: no go.work file found\n\t(run 'go work init' first or specify path using GOWORK environment variable)") + } workGraph := modload.LoadModGraph(ctx, "") _ = workGraph diff --git a/libgo/go/cmd/go/internal/workcmd/use.go b/libgo/go/cmd/go/internal/workcmd/use.go index 852e5b9..1ee2d4e 100644 --- a/libgo/go/cmd/go/internal/workcmd/use.go +++ b/libgo/go/cmd/go/internal/workcmd/use.go @@ -10,7 +10,10 @@ import ( "cmd/go/internal/base" "cmd/go/internal/fsys" "cmd/go/internal/modload" + "cmd/go/internal/str" "context" + "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -39,47 +42,50 @@ func init() { cmdUse.Run = runUse // break init cycle base.AddModCommonFlags(&cmdUse.Flag) - base.AddWorkfileFlag(&cmdUse.Flag) } func runUse(ctx context.Context, cmd *base.Command, args []string) { - modload.InitWorkfile() - modload.ForceUseModules = true var gowork string modload.InitWorkfile() gowork = modload.WorkFilePath() + if gowork == "" { + base.Fatalf("go: no go.work file found\n\t(run 'go work init' first or specify path using GOWORK environment variable)") + } workFile, err := modload.ReadWorkFile(gowork) if err != nil { base.Fatalf("go: %v", err) } - - haveDirs := make(map[string]bool) - for _, dir := range workFile.Use { - haveDirs[filepath.Join(filepath.Dir(gowork), filepath.FromSlash(dir.Path))] = true + workDir := filepath.Dir(gowork) // Absolute, since gowork itself is absolute. + + haveDirs := make(map[string][]string) // absolute → original(s) + for _, use := range workFile.Use { + var abs string + if filepath.IsAbs(use.Path) { + abs = filepath.Clean(use.Path) + } else { + abs = filepath.Join(workDir, use.Path) + } + haveDirs[abs] = append(haveDirs[abs], use.Path) } - addDirs := make(map[string]bool) - removeDirs := make(map[string]bool) + // keepDirs maps each absolute path to keep to the literal string to use for + // that path (either an absolute or a relative path), or the empty string if + // all entries for the absolute path should be removed. + keepDirs := make(map[string]string) + + // lookDir updates the entry in keepDirs for the directory dir, + // which is either absolute or relative to the current working directory + // (not necessarily the directory containing the workfile). lookDir := func(dir string) { - absDir := filepath.Join(base.Cwd(), dir) - // If the path is absolute, keep it absolute. If it's relative, - // make it relative to the go.work file rather than the working directory. - if !filepath.IsAbs(dir) { - rel, err := filepath.Rel(filepath.Dir(gowork), absDir) - if err == nil { - dir = rel - } - } - fi, err := os.Stat(filepath.Join(dir, "go.mod")) + absDir, dir := pathRel(workDir, dir) + + fi, err := os.Stat(filepath.Join(absDir, "go.mod")) if err != nil { if os.IsNotExist(err) { - - if haveDirs[absDir] { - removeDirs[dir] = true - } + keepDirs[absDir] = "" return } base.Errorf("go: %v", err) @@ -89,31 +95,96 @@ func runUse(ctx context.Context, cmd *base.Command, args []string) { base.Errorf("go: %v is not regular", filepath.Join(dir, "go.mod")) } - if !haveDirs[absDir] { - addDirs[dir] = true + if dup := keepDirs[absDir]; dup != "" && dup != dir { + base.Errorf(`go: already added "%s" as "%s"`, dir, dup) } + keepDirs[absDir] = dir } for _, useDir := range args { - if *useR { - fsys.Walk(useDir, func(path string, info fs.FileInfo, err error) error { - if !info.IsDir() { - return nil + if !*useR { + lookDir(useDir) + continue + } + + // Add or remove entries for any subdirectories that still exist. + err := fsys.Walk(useDir, func(path string, info fs.FileInfo, err error) error { + if !info.IsDir() { + if info.Mode()&fs.ModeSymlink != 0 { + if target, err := fsys.Stat(path); err == nil && target.IsDir() { + fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) + } } - lookDir(path) return nil - }) - continue + } + lookDir(path) + return nil + }) + if err != nil && !errors.Is(err, os.ErrNotExist) { + base.Errorf("go: %v", err) } - lookDir(useDir) - } - for dir := range removeDirs { - workFile.DropUse(filepath.ToSlash(dir)) + // Remove entries for subdirectories that no longer exist. + // Because they don't exist, they will be skipped by Walk. + absArg, _ := pathRel(workDir, useDir) + for absDir, _ := range haveDirs { + if str.HasFilePathPrefix(absDir, absArg) { + if _, ok := keepDirs[absDir]; !ok { + keepDirs[absDir] = "" // Mark for deletion. + } + } + } } - for dir := range addDirs { - workFile.AddUse(filepath.ToSlash(dir), "") + + base.ExitIfErrors() + + for absDir, keepDir := range keepDirs { + nKept := 0 + for _, dir := range haveDirs[absDir] { + if dir == keepDir { // (note that dir is always non-empty) + nKept++ + } else { + workFile.DropUse(dir) + } + } + if keepDir != "" && nKept != 1 { + // If we kept more than one copy, delete them all. + // We'll recreate a unique copy with AddUse. + if nKept > 1 { + workFile.DropUse(keepDir) + } + workFile.AddUse(keepDir, "") + } } modload.UpdateWorkFile(workFile) modload.WriteWorkFile(gowork, workFile) } + +// pathRel returns the absolute and canonical forms of dir for use in a +// go.work file located in directory workDir. +// +// If dir is relative, it is intepreted relative to base.Cwd() +// and its canonical form is relative to workDir if possible. +// If dir is absolute or cannot be made relative to workDir, +// its canonical form is absolute. +// +// Canonical absolute paths are clean. +// Canonical relative paths are clean and slash-separated. +func pathRel(workDir, dir string) (abs, canonical string) { + if filepath.IsAbs(dir) { + abs = filepath.Clean(dir) + return abs, abs + } + + abs = filepath.Join(base.Cwd(), dir) + rel, err := filepath.Rel(workDir, abs) + if err != nil { + // The path can't be made relative to the go.work file, + // so it must be kept absolute instead. + return abs, abs + } + + // Normalize relative paths to use slashes, so that checked-in go.work + // files with relative paths within the repo are platform-independent. + return abs, filepath.ToSlash(rel) +} diff --git a/libgo/go/cmd/go/internal/workcmd/work.go b/libgo/go/cmd/go/internal/workcmd/work.go index 5bb0a2e..d3cc250 100644 --- a/libgo/go/cmd/go/internal/workcmd/work.go +++ b/libgo/go/cmd/go/internal/workcmd/work.go @@ -27,7 +27,7 @@ workspace that does not specify modules to be used cannot be used to do builds from local modules. go.work files are line-oriented. Each line holds a single directive, -made up of a keyword followed by aruments. For example: +made up of a keyword followed by arguments. For example: go 1.18 diff --git a/libgo/go/cmd/go/script_test.go b/libgo/go/cmd/go/script_test.go index 0fc4b0f..55a88e0 100644 --- a/libgo/go/cmd/go/script_test.go +++ b/libgo/go/cmd/go/script_test.go @@ -142,6 +142,8 @@ var extraEnvKeys = []string{ "SYSTEMROOT", // must be preserved on Windows to find DLLs; golang.org/issue/25210 "WINDIR", // must be preserved on Windows to be able to run PowerShell command; golang.org/issue/30711 "LD_LIBRARY_PATH", // must be preserved on Unix systems to find shared libraries + "LIBRARY_PATH", // allow override of non-standard static library paths + "C_INCLUDE_PATH", // allow override non-standard include paths "CC", // don't lose user settings when invoking cgo "GO_TESTING_GOTOOLS", // for gccgo testing "GCCGO", // for gccgo testing @@ -648,9 +650,9 @@ func (ts *testScript) doCmdCmp(want simpleStatus, args []string, env, quiet bool } case successOrFailure: if eq { - fmt.Fprintf(&ts.log, "%s and %s do not differ", name1, name2) + fmt.Fprintf(&ts.log, "%s and %s do not differ\n", name1, name2) } else { - fmt.Fprintf(&ts.log, "%s and %s differ", name1, name2) + fmt.Fprintf(&ts.log, "%s and %s differ\n", name1, name2) } default: ts.fatalf("unsupported: %v cmp", want) @@ -902,7 +904,7 @@ func (ts *testScript) cmdStale(want simpleStatus, args []string) { tmpl := "{{if .Error}}{{.ImportPath}}: {{.Error.Err}}{{else}}" switch want { case failure: - tmpl += "{{if .Stale}}{{.ImportPath}} is unexpectedly stale{{end}}" + tmpl += "{{if .Stale}}{{.ImportPath}} is unexpectedly stale: {{.StaleReason}}{{end}}" case success: tmpl += "{{if not .Stale}}{{.ImportPath}} is unexpectedly NOT stale{{end}}" default: diff --git a/libgo/go/cmd/go/testdata/script/build_internal.txt b/libgo/go/cmd/go/testdata/script/build_internal.txt index 25aa18c..5b786f2 100644 --- a/libgo/go/cmd/go/testdata/script/build_internal.txt +++ b/libgo/go/cmd/go/testdata/script/build_internal.txt @@ -10,8 +10,10 @@ stderr 'internal' # Test internal packages outside GOROOT are respected cd ../testinternal2 +env GO111MODULE=off ! go build -v . stderr 'p\.go:3:8: use of internal package .*internal/w not allowed' +env GO111MODULE='' [gccgo] skip # gccgo does not have GOROOT cd ../testinternal diff --git a/libgo/go/cmd/go/testdata/script/mod_download_partial.txt b/libgo/go/cmd/go/testdata/script/mod_download_partial.txt index 3a02fcd..617b1fd 100644 --- a/libgo/go/cmd/go/testdata/script/mod_download_partial.txt +++ b/libgo/go/cmd/go/testdata/script/mod_download_partial.txt @@ -15,12 +15,13 @@ cp empty $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.partial go mod verify # 'go list' should not load packages from the directory. -# NOTE: the message "directory $dir outside available modules" is reported -# for directories not in the main module, active modules in the module cache, -# or local replacements. In this case, the directory is in the right place, -# but it's incomplete, so 'go list' acts as if it's not an active module. +# NOTE: the message "directory $dir outside main module or its selected dependencies" +# is reported for directories not in the main module, active modules in the +# module cache, or local replacements. In this case, the directory is in the +# right place, but it's incomplete, so 'go list' acts as if it's not an +# active module. ! go list $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 -stderr 'outside available modules' +stderr 'outside main module or its selected dependencies' # 'go list -m' should not print the directory. go list -m -f '{{.Dir}}' rsc.io/quote diff --git a/libgo/go/cmd/go/testdata/script/mod_fs_patterns.txt b/libgo/go/cmd/go/testdata/script/mod_fs_patterns.txt index a20fefd..276d04e 100644 --- a/libgo/go/cmd/go/testdata/script/mod_fs_patterns.txt +++ b/libgo/go/cmd/go/testdata/script/mod_fs_patterns.txt @@ -51,11 +51,11 @@ stdout '^at$' # a package path. cd ../badat/bad@ ! go list . -stderr 'directory . outside available modules' +stderr 'directory . outside main module or its selected dependencies' ! go list $PWD -stderr 'directory . outside available modules' +stderr 'directory . outside main module or its selected dependencies' ! go list $PWD/... -stderr 'directory . outside available modules' +stderr 'directory . outside main module or its selected dependencies' -- x/go.mod -- module m diff --git a/libgo/go/cmd/go/testdata/script/mod_invalid_version.txt b/libgo/go/cmd/go/testdata/script/mod_invalid_version.txt index 428b8aa..8385b08 100644 --- a/libgo/go/cmd/go/testdata/script/mod_invalid_version.txt +++ b/libgo/go/cmd/go/testdata/script/mod_invalid_version.txt @@ -194,10 +194,10 @@ cp go.mod.orig go.mod go mod edit -require github.com/pierrec/lz4@v2.0.9-0.20190209155647-9a39efadad3d+incompatible cd outside ! go list -m github.com/pierrec/lz4 -stderr 'go: example.com@v0.0.0 requires\n\tgithub.com/pierrec/lz4@v2.0.9-0.20190209155647-9a39efadad3d\+incompatible: invalid version: \+incompatible suffix not allowed: module contains a go.mod file, so semantic import versioning is required' +stderr '^go: example.com@v0.0.0 requires\n\tgithub.com/pierrec/lz4@v2.0.9-0.20190209155647-9a39efadad3d\+incompatible: invalid version: module contains a go.mod file, so module path must match major version \("github.com/pierrec/lz4/v2"\)$' cd .. ! go list -m github.com/pierrec/lz4 -stderr 'github.com/pierrec/lz4@v2.0.9-0.20190209155647-9a39efadad3d\+incompatible: invalid version: \+incompatible suffix not allowed: module contains a go.mod file, so semantic import versioning is required' +stderr '^go: github.com/pierrec/lz4@v2.0.9-0.20190209155647-9a39efadad3d\+incompatible: invalid version: module contains a go.mod file, so module path must match major version \("github.com/pierrec/lz4/v2"\)$' # A +incompatible pseudo-version is valid for a revision of the module # that lacks a go.mod file. @@ -222,7 +222,7 @@ stdout 'github.com/pierrec/lz4 v2.0.5\+incompatible' # not resolve to a pseudo-version with a different major version. cp go.mod.orig go.mod ! go get github.com/pierrec/lz4@v2.0.8 -stderr 'go: github.com/pierrec/lz4@v2.0.8: invalid version: module contains a go.mod file, so major version must be compatible: should be v0 or v1, not v2' +stderr 'go: github.com/pierrec/lz4@v2.0.8: invalid version: module contains a go.mod file, so module path must match major version \("github.com/pierrec/lz4/v2"\)$' # An invalid +incompatible suffix for a canonical version should error out, # not resolve to a pseudo-version. @@ -233,10 +233,10 @@ cp go.mod.orig go.mod go mod edit -require github.com/pierrec/lz4@v2.0.8+incompatible cd outside ! go list -m github.com/pierrec/lz4 -stderr 'github.com/pierrec/lz4@v2.0.8\+incompatible: invalid version: \+incompatible suffix not allowed: module contains a go.mod file, so semantic import versioning is required' +stderr '^go: github.com/pierrec/lz4@v2.0.8\+incompatible: invalid version: module contains a go.mod file, so module path must match major version \("github.com/pierrec/lz4/v2"\)$' cd .. ! go list -m github.com/pierrec/lz4 -stderr 'github.com/pierrec/lz4@v2.0.8\+incompatible: invalid version: \+incompatible suffix not allowed: module contains a go.mod file, so semantic import versioning is required' +stderr '^go: github.com/pierrec/lz4@v2.0.8\+incompatible: invalid version: module contains a go.mod file, so module path must match major version \("github.com/pierrec/lz4/v2"\)$' -- go.mod.orig -- module example.com diff --git a/libgo/go/cmd/go/testdata/script/mod_list_dir.txt b/libgo/go/cmd/go/testdata/script/mod_list_dir.txt index 7ad65ff..157d3b6 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_dir.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_dir.txt @@ -24,7 +24,7 @@ go get rsc.io/sampler@v1.3.1 go list -f '{{.ImportPath}}' $GOPATH/pkg/mod/rsc.io/sampler@v1.3.1 stdout '^rsc.io/sampler$' ! go list -f '{{.ImportPath}}' $GOPATH/pkg/mod/rsc.io/sampler@v1.3.0 -stderr 'outside available modules' +stderr 'outside main module or its selected dependencies' -- go.mod -- module x diff --git a/libgo/go/cmd/go/testdata/script/mod_list_replace_dir.txt b/libgo/go/cmd/go/testdata/script/mod_list_replace_dir.txt index eac5ca7..b446543 100644 --- a/libgo/go/cmd/go/testdata/script/mod_list_replace_dir.txt +++ b/libgo/go/cmd/go/testdata/script/mod_list_replace_dir.txt @@ -9,7 +9,7 @@ go get go mod download rsc.io/quote@v1.5.2 ! go list $GOPATH/pkg/mod/rsc.io/quote@v1.5.2 -stderr '^directory ..[/\\]pkg[/\\]mod[/\\]rsc.io[/\\]quote@v1.5.2 outside available modules$' +stderr '^directory ..[/\\]pkg[/\\]mod[/\\]rsc.io[/\\]quote@v1.5.2 outside main module or its selected dependencies$' go list $GOPATH/pkg/mod/rsc.io/quote@v1.5.1 stdout 'rsc.io/quote' diff --git a/libgo/go/cmd/go/testdata/script/run_issue51125.txt b/libgo/go/cmd/go/testdata/script/run_issue51125.txt new file mode 100644 index 0000000..8fa4486 --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/run_issue51125.txt @@ -0,0 +1,54 @@ +# Regression test for https://go.dev/issue/51125: +# Relative import paths (a holdover from GOPATH) were accidentally allowed in module mode. + +cd $WORK + +# Relative imports should not be allowed with a go.mod file. + +! go run driver.go +stderr '^driver.go:3:8: "./mypkg" is relative, but relative import paths are not supported in module mode$' + +go list -e -f '{{with .Error}}{{.}}{{end}}' -deps driver.go +stdout '^driver.go:3:8: "./mypkg" is relative, but relative import paths are not supported in module mode$' +! stderr . + + +# Relative imports should not be allowed in module mode even without a go.mod file. +rm go.mod + +! go run driver.go +stderr '^driver.go:3:8: "./mypkg" is relative, but relative import paths are not supported in module mode$' + +go list -e -f '{{with .Error}}{{.}}{{end}}' -deps driver.go +stdout '^driver.go:3:8: "./mypkg" is relative, but relative import paths are not supported in module mode$' +! stderr . + + +# In GOPATH mode, they're still allowed (but only outside of GOPATH/src). +env GO111MODULE=off + +[!short] go run driver.go + +go list -deps driver.go + + +-- $WORK/go.mod -- +module example + +go 1.17 +-- $WORK/driver.go -- +package main + +import "./mypkg" + +func main() { + mypkg.MyFunc() +} +-- $WORK/mypkg/code.go -- +package mypkg + +import "fmt" + +func MyFunc() { + fmt.Println("Hello, world!") +} diff --git a/libgo/go/cmd/go/testdata/script/test_fuzz_dup_cache.txt b/libgo/go/cmd/go/testdata/script/test_fuzz_dup_cache.txt new file mode 100644 index 0000000..52d44a2 --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/test_fuzz_dup_cache.txt @@ -0,0 +1,52 @@ +[!fuzz] skip +[short] skip + +# This test checks that cached corpus loading properly handles duplicate entries (this can +# happen when a f.Add value has a duplicate entry in the cached corpus.) Duplicate entries +# should be discarded, and the rest of the cache should be loaded as normal. + +env GOCACHE=$WORK/cache +env GODEBUG=fuzzdebug=1 + +mkdir -p $GOCACHE/fuzz/fuzztest/FuzzTarget +go run ./populate $GOCACHE/fuzz/fuzztest/FuzzTarget + +go test -fuzz=FuzzTarget -fuzztime=10x . +stdout 'entries: 5' + +-- go.mod -- +module fuzztest + +go 1.17 + +-- fuzz_test.go -- +package fuzz + +import "testing" + +func FuzzTarget(f *testing.F) { + f.Add(int(0)) + f.Fuzz(func(t *testing.T, _ int) {}) +} + +-- populate/main.go -- +package main + +import ( + "path/filepath" + "fmt" + "os" +) + +func main() { + for i := 0; i < 10; i++ { + b := byte(0) + if i > 5 { + b = byte(i) + } + tmpl := "go test fuzz v1\nint(%d)\n" + if err := os.WriteFile(filepath.Join(os.Args[1], fmt.Sprint(i)), []byte(fmt.Sprintf(tmpl, b)), 0777); err != nil { + panic(err) + } + } +} \ No newline at end of file diff --git a/libgo/go/cmd/go/testdata/script/test_fuzz_return.txt b/libgo/go/cmd/go/testdata/script/test_fuzz_return.txt new file mode 100644 index 0000000..63275aa --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/test_fuzz_return.txt @@ -0,0 +1,19 @@ +[short] skip + +! go test . +stdout '^panic: testing: fuzz target must not return a value \[recovered\]$' + +-- go.mod -- +module test +go 1.18 +-- x_test.go -- +package test + +import "testing" + +func FuzzReturnErr(f *testing.F) { + f.Add("hello, validation!") + f.Fuzz(func(t *testing.T, in string) string { + return in + }) +} diff --git a/libgo/go/cmd/go/testdata/script/test_relative_cmdline.txt b/libgo/go/cmd/go/testdata/script/test_relative_cmdline.txt index 37aa3b1..4c1fdf9 100644 --- a/libgo/go/cmd/go/testdata/script/test_relative_cmdline.txt +++ b/libgo/go/cmd/go/testdata/script/test_relative_cmdline.txt @@ -1,5 +1,7 @@ # Relative imports in command line package +env GO111MODULE=off + # Run tests outside GOPATH. env GO111MODULE=off env GOPATH=$WORK/tmp diff --git a/libgo/go/cmd/go/testdata/script/work.txt b/libgo/go/cmd/go/testdata/script/work.txt index cbb3746..a10bf5a 100644 --- a/libgo/go/cmd/go/testdata/script/work.txt +++ b/libgo/go/cmd/go/testdata/script/work.txt @@ -32,7 +32,9 @@ stdout 'example.com/b' go list -mod=readonly all ! go list -mod=mod all stderr '^go: -mod may only be set to readonly when in workspace mode' -go list -mod=mod -workfile=off all +env GOWORK=off +go list -mod=mod all +env GOWORK= # Test that duplicates in the use list return an error cp go.work go.work.backup @@ -53,7 +55,9 @@ go run example.com/d # This exercises the code that determines which module command-line-arguments # belongs to. go list ./b/main.go -go build -n -workfile=off -o foo foo.go +env GOWORK=off +go build -n -o foo foo.go +env GOWORK= go build -n -o foo foo.go -- go.work.dup -- diff --git a/libgo/go/cmd/go/testdata/script/work_edit.txt b/libgo/go/cmd/go/testdata/script/work_edit.txt index fd04bbd..71959ca 100644 --- a/libgo/go/cmd/go/testdata/script/work_edit.txt +++ b/libgo/go/cmd/go/testdata/script/work_edit.txt @@ -30,7 +30,8 @@ cmp stdout go.work.want_print go work edit -json -go 1.19 -use b -dropuse c -replace 'x.1@v1.4.0 = ../z' -dropreplace x.1 -dropreplace x.1@v1.3.0 cmp stdout go.work.want_json -go work edit -print -fmt -workfile $GOPATH/src/unformatted +env GOWORK=$GOPATH/src/unformatted +go work edit -print -fmt cmp stdout formatted -- m/go.mod -- diff --git a/libgo/go/cmd/go/testdata/script/work_env.txt b/libgo/go/cmd/go/testdata/script/work_env.txt index ec3d3be..511bb4e 100644 --- a/libgo/go/cmd/go/testdata/script/work_env.txt +++ b/libgo/go/cmd/go/testdata/script/work_env.txt @@ -13,6 +13,10 @@ cd src go env GOWORK stdout 'go.work' +env GOWORK='off' +go env GOWORK +stdout 'off' + ! go env -w GOWORK=off stderr '^go: GOWORK cannot be modified$' diff --git a/libgo/go/cmd/go/testdata/script/work_gowork.txt b/libgo/go/cmd/go/testdata/script/work_gowork.txt new file mode 100644 index 0000000..1cfbf0c --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/work_gowork.txt @@ -0,0 +1,24 @@ +env GOWORK=stop.work +! go list a # require absolute path +! stderr panic +env GOWORK=doesnotexist +! go list a +! stderr panic + +env GOWORK=$GOPATH/src/stop.work +go list -n a +go build -n a +go test -n a + +-- stop.work -- +go 1.18 + +use ./a +-- a/a.go -- +package a +-- a/a_test.go -- +package a +-- a/go.mod -- +module a + +go 1.18 \ No newline at end of file diff --git a/libgo/go/cmd/go/testdata/script/work_init_gowork.txt b/libgo/go/cmd/go/testdata/script/work_init_gowork.txt new file mode 100644 index 0000000..55ac99b --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/work_init_gowork.txt @@ -0,0 +1,19 @@ +# Test that the GOWORK environment variable flag is used by go work init. + +! exists go.work +go work init +exists go.work + +env GOWORK=$GOPATH/src/foo/foo.work +! exists foo/foo.work +go work init +exists foo/foo.work + +env GOWORK= +cd foo/bar +! go work init +stderr 'already exists' + +# Create directories to make go.work files in. +-- foo/dummy.txt -- +-- foo/bar/dummy.txt -- diff --git a/libgo/go/cmd/go/testdata/script/work_issue51204.txt b/libgo/go/cmd/go/testdata/script/work_issue51204.txt new file mode 100644 index 0000000..d483002 --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/work_issue51204.txt @@ -0,0 +1,57 @@ +go work sync + +go list -f '{{.Dir}}' example.com/test +stdout '^'$PWD${/}test'$' + +-- go.work -- +go 1.18 + +use ( + ./test2 + ./test2/sub +) +-- test/go.mod -- +module example.com/test + +go 1.18 +-- test/file.go -- +package test + +func DoSomething() { +} +-- test2/go.mod -- +module example.com/test2 + +go 1.18 + +replace example.com/test => ../test + +require example.com/test v0.0.0-00010101000000-000000000000 +-- test2/file.go -- +package test2 + +import ( + "example.com/test" +) + +func DoSomething() { + test.DoSomething() +} +-- test2/sub/go.mod -- +module example.com/test2/sub + +go 1.18 + +replace example.com/test => ../../test + +require example.com/test v0.0.0 +-- test2/sub/file.go -- +package test2 + +import ( + "example.com/test" +) + +func DoSomething() { + test.DoSomething() +} diff --git a/libgo/go/cmd/go/testdata/script/work_module_not_in_go_work.txt b/libgo/go/cmd/go/testdata/script/work_module_not_in_go_work.txt new file mode 100644 index 0000000..23d908c --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/work_module_not_in_go_work.txt @@ -0,0 +1,25 @@ +# This is a regression test for issue #49632. +# The Go command should mention go.work if the user +# tries to load a local package that's in a module +# that's not in go.work and can't be resolved. + +! go list ./... +stderr 'pattern ./...: directory prefix . does not contain modules listed in go.work or their selected dependencies' + +! go list ./a +stderr 'directory a outside modules listed in go.work' + +-- go.work -- +go 1.18 + +use ./b +-- a/go.mod -- +module example.com/a + +go 1.18 +-- a/a.go -- +package a +-- b/go.mod -- +module example.com/b + +go 1.18 diff --git a/libgo/go/cmd/go/testdata/script/work_nowork.txt b/libgo/go/cmd/go/testdata/script/work_nowork.txt new file mode 100644 index 0000000..b4c9b1d --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/work_nowork.txt @@ -0,0 +1,20 @@ +! go work use +stderr '^go: no go\.work file found\n\t\(run ''go work init'' first or specify path using GOWORK environment variable\)$' + +! go work use . +stderr '^go: no go\.work file found\n\t\(run ''go work init'' first or specify path using GOWORK environment variable\)$' + +! go work edit +stderr '^go: no go\.work file found\n\t\(run ''go work init'' first or specify path using GOWORK environment variable\)$' + +! go work edit -go=1.18 +stderr '^go: no go\.work file found\n\t\(run ''go work init'' first or specify path using GOWORK environment variable\)$' + +! go work sync +stderr '^go: no go\.work file found\n\t\(run ''go work init'' first or specify path using GOWORK environment variable\)$' + +-- go.mod -- +module example +go 1.18 +-- README.txt -- +There is no go.work file here. diff --git a/libgo/go/cmd/go/testdata/script/work_replace_conflict.txt b/libgo/go/cmd/go/testdata/script/work_replace_conflict.txt index 81d1fcb..7b71b0f 100644 --- a/libgo/go/cmd/go/testdata/script/work_replace_conflict.txt +++ b/libgo/go/cmd/go/testdata/script/work_replace_conflict.txt @@ -2,7 +2,7 @@ # overriding it in the go.work file. ! go list -m example.com/dep -stderr 'go: conflicting replacements for example.com/dep@v1.0.0:\n\t./dep1\n\t./dep2\nuse "go work edit -replace example.com/dep@v1.0.0=\[override\]" to resolve' +stderr 'go: conflicting replacements for example.com/dep@v1.0.0:\n\t'$PWD${/}'dep1\n\t'$PWD${/}'dep2\nuse "go work edit -replace example.com/dep@v1.0.0=\[override\]" to resolve' go work edit -replace example.com/dep@v1.0.0=./dep1 go list -m example.com/dep stdout 'example.com/dep v1.0.0 => ./dep1' @@ -15,7 +15,7 @@ use n module example.com/m require example.com/dep v1.0.0 -replace example.com/dep v1.0.0 => ./dep1 +replace example.com/dep v1.0.0 => ../dep1 -- m/m.go -- package m @@ -28,7 +28,7 @@ func F() { module example.com/n require example.com/dep v1.0.0 -replace example.com/dep v1.0.0 => ./dep2 +replace example.com/dep v1.0.0 => ../dep2 -- n/n.go -- package n diff --git a/libgo/go/cmd/go/testdata/script/work_use_deleted.txt b/libgo/go/cmd/go/testdata/script/work_use_deleted.txt new file mode 100644 index 0000000..660eb56 --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/work_use_deleted.txt @@ -0,0 +1,22 @@ +go work use -r . +cmp go.work go.work.want + +-- go.work -- +go 1.18 + +use ( + . + sub + sub/dir/deleted +) +-- go.work.want -- +go 1.18 + +use sub/dir +-- sub/README.txt -- +A go.mod file has been deleted from this directory. +In addition, the entire subdirectory sub/dir/deleted +has been deleted, along with sub/dir/deleted/go.mod. +-- sub/dir/go.mod -- +module example/sub/dir +go 1.18 diff --git a/libgo/go/cmd/go/testdata/script/work_use_dot.txt b/libgo/go/cmd/go/testdata/script/work_use_dot.txt new file mode 100644 index 0000000..ccd83d6 --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/work_use_dot.txt @@ -0,0 +1,49 @@ +cp go.work go.work.orig + +# If the current directory contains a go.mod file, +# 'go work use .' should add an entry for it. +cd bar/baz +go work use . +cmp ../../go.work ../../go.work.rel + +# If the current directory lacks a go.mod file, 'go work use .' +# should remove its entry. +mv go.mod go.mod.bak +go work use . +cmp ../../go.work ../../go.work.orig + +# If the path is absolute, it should remain absolute. +mv go.mod.bak go.mod +go work use $PWD +grep -count=1 '^use ' ../../go.work +grep '^use ["]?'$PWD'["]?$' ../../go.work + +# An absolute path should replace an entry for the corresponding relative path +# and vice-versa. +go work use . +cmp ../../go.work ../../go.work.rel +go work use $PWD +grep -count=1 '^use ' ../../go.work +grep '^use ["]?'$PWD'["]?$' ../../go.work + +# If both the absolute and relative paths are named, 'go work use' should error +# out: we don't know which one to use, and shouldn't add both because the +# resulting workspace would contain a duplicate module. +cp ../../go.work.orig ../../go.work +! go work use $PWD . +stderr '^go: already added "bar/baz" as "'$PWD'"$' +cmp ../../go.work ../../go.work.orig + + +-- go.mod -- +module example +go 1.18 +-- go.work -- +go 1.18 +-- go.work.rel -- +go 1.18 + +use bar/baz +-- bar/baz/go.mod -- +module example/bar/baz +go 1.18 diff --git a/libgo/go/cmd/go/testdata/script/work_use_issue50958.txt b/libgo/go/cmd/go/testdata/script/work_use_issue50958.txt new file mode 100644 index 0000000..7a25531 --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/work_use_issue50958.txt @@ -0,0 +1,17 @@ +go work use -r . +cmp go.work go.work.want + +-- go.mod -- +module example +go 1.18 +-- go.work -- +go 1.18 + +use sub +-- go.work.want -- +go 1.18 + +use . +-- sub/README.txt -- +This directory no longer contains a go.mod file. + diff --git a/libgo/go/cmd/go/testdata/script/work_vet.txt b/libgo/go/cmd/go/testdata/script/work_vet.txt new file mode 100644 index 0000000..e258fc0 --- /dev/null +++ b/libgo/go/cmd/go/testdata/script/work_vet.txt @@ -0,0 +1,19 @@ +! go vet ./a +stderr 'fmt.Println call has possible formatting directive' + +-- go.work -- +go 1.18 + +use ./a +-- a/go.mod -- +module example.com/a + +go 1.18 +-- a/a.go -- +package a + +import "fmt" + +func A() { + fmt.Println("%s") +} \ No newline at end of file diff --git a/libgo/go/cmd/go/testdata/script/work_workfile.txt b/libgo/go/cmd/go/testdata/script/work_workfile.txt deleted file mode 100644 index b629181..0000000 --- a/libgo/go/cmd/go/testdata/script/work_workfile.txt +++ /dev/null @@ -1,21 +0,0 @@ -! go list -workfile=stop.work a # require absolute path -! stderr panic -! go list -workfile=doesnotexist a -! stderr panic - -go list -n -workfile=$GOPATH/src/stop.work a -go build -n -workfile=$GOPATH/src/stop.work a -go test -n -workfile=$GOPATH/src/stop.work a - --- stop.work -- -go 1.18 - -use ./a --- a/a.go -- -package a --- a/a_test.go -- -package a --- a/go.mod -- -module a - -go 1.18 \ No newline at end of file diff --git a/libgo/go/cmd/gofmt/gofmt.go b/libgo/go/cmd/gofmt/gofmt.go index 51f6e65..4280ed4 100644 --- a/libgo/go/cmd/gofmt/gofmt.go +++ b/libgo/go/cmd/gofmt/gofmt.go @@ -52,6 +52,16 @@ const ( printerNormalizeNumbers = 1 << 30 ) +// fdSem guards the number of concurrently-open file descriptors. +// +// For now, this is arbitrarily set to 200, based on the observation that many +// platforms default to a kernel limit of 256. Ideally, perhaps we should derive +// it from rlimit on platforms that support that system call. +// +// File descriptors opened from outside of this package are not tracked, +// so this limit may be approximate. +var fdSem = make(chan bool, 200) + var ( rewrite func(*token.FileSet, *ast.File) *ast.File parserMode parser.Mode @@ -213,51 +223,9 @@ func (r *reporter) ExitCode() int { // If info == nil, we are formatting stdin instead of a file. // If in == nil, the source is the contents of the file with the given filename. func processFile(filename string, info fs.FileInfo, in io.Reader, r *reporter) error { - if in == nil { - var err error - in, err = os.Open(filename) - if err != nil { - return err - } - } - - // Compute the file's size and read its contents with minimal allocations. - // - // If the size is unknown (or bogus, or overflows an int), fall back to - // a size-independent ReadAll. - var src []byte - size := -1 - if info != nil && info.Mode().IsRegular() && int64(int(info.Size())) == info.Size() { - size = int(info.Size()) - } - if size+1 > 0 { - // If we have the FileInfo from filepath.WalkDir, use it to make - // a buffer of the right size and avoid ReadAll's reallocations. - // - // We try to read size+1 bytes so that we can detect modifications: if we - // read more than size bytes, then the file was modified concurrently. - // (If that happens, we could, say, append to src to finish the read, or - // proceed with a truncated buffer — but the fact that it changed at all - // indicates a possible race with someone editing the file, so we prefer to - // stop to avoid corrupting it.) - src = make([]byte, size+1) - n, err := io.ReadFull(in, src) - if err != nil && err != io.ErrUnexpectedEOF { - return err - } - if n < size { - return fmt.Errorf("error: size of %s changed during reading (from %d to %d bytes)", filename, size, n) - } else if n > size { - return fmt.Errorf("error: size of %s changed during reading (from %d to >=%d bytes)", filename, size, len(src)) - } - src = src[:n] - } else { - // The file is not known to be regular, so we don't have a reliable size for it. - var err error - src, err = io.ReadAll(in) - if err != nil { - return err - } + src, err := readFile(filename, info, in) + if err != nil { + return err } fileSet := token.NewFileSet() @@ -306,7 +274,9 @@ func processFile(filename string, info fs.FileInfo, in io.Reader, r *reporter) e if err != nil { return err } + fdSem <- true err = os.WriteFile(filename, res, perm) + <-fdSem if err != nil { os.Rename(bakname, filename) return err @@ -333,6 +303,65 @@ func processFile(filename string, info fs.FileInfo, in io.Reader, r *reporter) e return err } +// readFile reads the contents of filename, described by info. +// If in is non-nil, readFile reads directly from it. +// Otherwise, readFile opens and reads the file itself, +// with the number of concurrently-open files limited by fdSem. +func readFile(filename string, info fs.FileInfo, in io.Reader) ([]byte, error) { + if in == nil { + fdSem <- true + var err error + f, err := os.Open(filename) + if err != nil { + return nil, err + } + in = f + defer func() { + f.Close() + <-fdSem + }() + } + + // Compute the file's size and read its contents with minimal allocations. + // + // If we have the FileInfo from filepath.WalkDir, use it to make + // a buffer of the right size and avoid ReadAll's reallocations. + // + // If the size is unknown (or bogus, or overflows an int), fall back to + // a size-independent ReadAll. + size := -1 + if info != nil && info.Mode().IsRegular() && int64(int(info.Size())) == info.Size() { + size = int(info.Size()) + } + if size+1 <= 0 { + // The file is not known to be regular, so we don't have a reliable size for it. + var err error + src, err := io.ReadAll(in) + if err != nil { + return nil, err + } + return src, nil + } + + // We try to read size+1 bytes so that we can detect modifications: if we + // read more than size bytes, then the file was modified concurrently. + // (If that happens, we could, say, append to src to finish the read, or + // proceed with a truncated buffer — but the fact that it changed at all + // indicates a possible race with someone editing the file, so we prefer to + // stop to avoid corrupting it.) + src := make([]byte, size+1) + n, err := io.ReadFull(in, src) + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } + if n < size { + return nil, fmt.Errorf("error: size of %s changed during reading (from %d to %d bytes)", filename, size, n) + } else if n > size { + return nil, fmt.Errorf("error: size of %s changed during reading (from %d to >=%d bytes)", filename, size, len(src)) + } + return src[:n], nil +} + func main() { // Arbitrarily limit in-flight work to 2MiB times the number of threads. // @@ -354,12 +383,16 @@ func gofmtMain(s *sequencer) { flag.Parse() if *cpuprofile != "" { + fdSem <- true f, err := os.Create(*cpuprofile) if err != nil { s.AddReport(fmt.Errorf("creating cpu profile: %s", err)) return } - defer f.Close() + defer func() { + f.Close() + <-fdSem + }() pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } @@ -474,6 +507,9 @@ const chmodSupported = runtime.GOOS != "windows" // with