From d79a22eddc694970316992927c669dd801e07557 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 8 Apr 2020 16:51:01 -0700 Subject: libgo: update to final 1.14.2 release Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/227551 --- libgo/go/cmd/go/alldocs.go | 52 +++++++++++------- libgo/go/cmd/go/internal/modcmd/verify.go | 11 ++-- libgo/go/cmd/go/internal/modfetch/cache.go | 40 ++++++++++++-- libgo/go/cmd/go/internal/modfetch/fetch.go | 61 ++++++++++++++-------- libgo/go/cmd/go/internal/modload/build.go | 4 +- libgo/go/cmd/go/internal/modload/help.go | 52 +++++++++++------- .../go/cmd/go/internal/robustio/robustio_flaky.go | 2 +- libgo/go/runtime/mbitmap.go | 6 ++- libgo/go/runtime/stubs.go | 7 +++ 9 files changed, 162 insertions(+), 73 deletions(-) (limited to 'libgo/go') diff --git a/libgo/go/cmd/go/alldocs.go b/libgo/go/cmd/go/alldocs.go index 971a756..c2678c3 100644 --- a/libgo/go/cmd/go/alldocs.go +++ b/libgo/go/cmd/go/alldocs.go @@ -2506,13 +2506,21 @@ // The "go get" command remains permitted to update go.mod even with -mod=readonly, // and the "go mod" commands do not take the -mod flag (or any other build flags). // -// If invoked with -mod=vendor, the go command assumes that the vendor -// directory holds the correct copies of dependencies and ignores -// the dependency descriptions in go.mod. +// If invoked with -mod=vendor, the go command loads packages from the main +// module's vendor directory instead of downloading modules to and loading packages +// from the module cache. The go command assumes the vendor directory holds +// correct copies of dependencies, and it does not compute the set of required +// module versions from go.mod files. However, the go command does check that +// vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent +// with go.mod. // // If invoked with -mod=mod, the go command loads modules from the module cache // even if there is a vendor directory present. // +// If the go command is not invoked with a -mod flag and the vendor directory +// is present and the "go" version in go.mod is 1.14 or higher, the go command +// will act as if it were invoked with -mod=vendor. +// // Pseudo-versions // // The go.mod file and the go command more generally use semantic versions as @@ -2710,22 +2718,28 @@ // // Modules and vendoring // -// When using modules, the go command completely ignores vendor directories. -// -// By default, the go command satisfies dependencies by downloading modules -// from their sources and using those downloaded copies (after verification, -// as described in the previous section). To allow interoperation with older -// versions of Go, or to ensure that all files used for a build are stored -// together in a single file tree, 'go mod vendor' creates a directory named -// vendor in the root directory of the main module and stores there all the -// packages from dependency modules that are needed to support builds and -// tests of packages in the main module. -// -// To build using the main module's top-level vendor directory to satisfy -// dependencies (disabling use of the usual network sources and local -// caches), use 'go build -mod=vendor'. Note that only the main module's -// top-level vendor directory is used; vendor directories in other locations -// are still ignored. +// When using modules, the go command typically satisfies dependencies by +// downloading modules from their sources and using those downloaded copies +// (after verification, as described in the previous section). Vendoring may +// be used to allow interoperation with older versions of Go, or to ensure +// that all files used for a build are stored together in a single file tree. +// +// The command 'go mod vendor' constructs a directory named vendor in the main +// module's root directory that contains copies of all packages needed to support +// builds and tests of packages in the main module. 'go mod vendor' also +// creates the file vendor/modules.txt that contains metadata about vendored +// packages and module versions. This file should be kept consistent with go.mod: +// when vendoring is used, 'go mod vendor' should be run after go.mod is updated. +// +// If the vendor directory is present in the main module's root directory, it will +// be used automatically if the "go" version in the main module's go.mod file is +// 1.14 or higher. Build commands like 'go build' and 'go test' will load packages +// from the vendor directory instead of accessing the network or the local module +// cache. To explicitly enable vendoring, invoke the go command with the flag +// -mod=vendor. To disable vendoring, use the flag -mod=mod. +// +// Unlike vendoring in GOPATH, the go command ignores vendor directories in +// locations other than the main module's root directory. // // // Module authentication using go.sum diff --git a/libgo/go/cmd/go/internal/modcmd/verify.go b/libgo/go/cmd/go/internal/modcmd/verify.go index 831e5cf..ac3f135 100644 --- a/libgo/go/cmd/go/internal/modcmd/verify.go +++ b/libgo/go/cmd/go/internal/modcmd/verify.go @@ -6,6 +6,7 @@ package modcmd import ( "bytes" + "errors" "fmt" "io/ioutil" "os" @@ -67,12 +68,10 @@ func verifyMod(mod module.Version) bool { _, zipErr = os.Stat(zip) } dir, dirErr := modfetch.DownloadDir(mod) - if dirErr == nil { - _, dirErr = os.Stat(dir) - } data, err := ioutil.ReadFile(zip + "hash") if err != nil { - if zipErr != nil && os.IsNotExist(zipErr) && dirErr != nil && os.IsNotExist(dirErr) { + if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) && + dirErr != nil && errors.Is(dirErr, os.ErrNotExist) { // Nothing downloaded yet. Nothing to verify. return true } @@ -81,7 +80,7 @@ func verifyMod(mod module.Version) bool { } h := string(bytes.TrimSpace(data)) - if zipErr != nil && os.IsNotExist(zipErr) { + if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) { // ok } else { hZ, err := dirhash.HashZip(zip, dirhash.DefaultHash) @@ -93,7 +92,7 @@ func verifyMod(mod module.Version) bool { ok = false } } - if dirErr != nil && os.IsNotExist(dirErr) { + if dirErr != nil && errors.Is(dirErr, os.ErrNotExist) { // ok } else { hD, err := dirhash.HashDir(dir, mod.Path+"@"+mod.Version, dirhash.DefaultHash) diff --git a/libgo/go/cmd/go/internal/modfetch/cache.go b/libgo/go/cmd/go/internal/modfetch/cache.go index 947192b..d6ff068 100644 --- a/libgo/go/cmd/go/internal/modfetch/cache.go +++ b/libgo/go/cmd/go/internal/modfetch/cache.go @@ -7,6 +7,7 @@ package modfetch import ( "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -56,8 +57,11 @@ func CachePath(m module.Version, suffix string) (string, error) { return filepath.Join(dir, encVer+"."+suffix), nil } -// DownloadDir returns the directory to which m should be downloaded. -// Note that the directory may not yet exist. +// DownloadDir returns the directory to which m should have been downloaded. +// An error will be returned if the module path or version cannot be escaped. +// An error satisfying errors.Is(err, os.ErrNotExist) will be returned +// along with the directory if the directory does not exist or if the directory +// is not completely populated. func DownloadDir(m module.Version) (string, error) { if PkgMod == "" { return "", fmt.Errorf("internal error: modfetch.PkgMod not set") @@ -76,9 +80,39 @@ func DownloadDir(m module.Version) (string, error) { if err != nil { return "", err } - return filepath.Join(PkgMod, enc+"@"+encVer), nil + + dir := filepath.Join(PkgMod, enc+"@"+encVer) + if fi, err := os.Stat(dir); os.IsNotExist(err) { + return dir, err + } else if err != nil { + return dir, &DownloadDirPartialError{dir, err} + } else if !fi.IsDir() { + return dir, &DownloadDirPartialError{dir, errors.New("not a directory")} + } + partialPath, err := CachePath(m, "partial") + if err != nil { + return dir, err + } + if _, err := os.Stat(partialPath); err == nil { + return dir, &DownloadDirPartialError{dir, errors.New("not completely extracted")} + } else if !os.IsNotExist(err) { + return dir, err + } + return dir, nil +} + +// DownloadDirPartialError is returned by DownloadDir if a module directory +// exists but was not completely populated. +// +// DownloadDirPartialError is equivalent to os.ErrNotExist. +type DownloadDirPartialError struct { + Dir string + Err error } +func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) } +func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist } + // lockVersion locks a file within the module cache that guards the downloading // and extraction of the zipfile for the given module version. func lockVersion(mod module.Version) (unlock func(), err error) { diff --git a/libgo/go/cmd/go/internal/modfetch/fetch.go b/libgo/go/cmd/go/internal/modfetch/fetch.go index 54fbd92..aadf883 100644 --- a/libgo/go/cmd/go/internal/modfetch/fetch.go +++ b/libgo/go/cmd/go/internal/modfetch/fetch.go @@ -22,6 +22,7 @@ import ( "cmd/go/internal/lockedfile" "cmd/go/internal/par" "cmd/go/internal/renameio" + "cmd/go/internal/robustio" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -45,24 +46,27 @@ func Download(mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := DownloadDir(mod) + dir, err := download(mod) if err != nil { return cached{"", err} } - if err := download(mod, dir); err != nil { - return cached{"", err} - } checkMod(mod) return cached{dir, nil} }).(cached) return c.dir, c.err } -func download(mod module.Version, dir string) (err error) { - // If the directory exists, the module has already been extracted. - fi, err := os.Stat(dir) - if err == nil && fi.IsDir() { - return nil +func download(mod module.Version) (dir string, err error) { + // If the directory exists, and no .partial file exists, + // the module has already been completely extracted. + // .partial files may be created when future versions of cmd/go + // extract module zip directories in place instead of extracting + // to a random temporary directory and renaming. + dir, err = DownloadDir(mod) + if err == nil { + return dir, nil + } else if dir == "" || !errors.Is(err, os.ErrNotExist) { + return "", err } // To avoid cluttering the cache with extraneous files, @@ -70,22 +74,24 @@ func download(mod module.Version, dir string) (err error) { // Invoke DownloadZip before locking the file. zipfile, err := DownloadZip(mod) if err != nil { - return err + return "", err } unlock, err := lockVersion(mod) if err != nil { - return err + return "", err } defer unlock() // Check whether the directory was populated while we were waiting on the lock. - fi, err = os.Stat(dir) - if err == nil && fi.IsDir() { - return nil + _, dirErr := DownloadDir(mod) + if dirErr == nil { + return dir, nil } + _, dirExists := dirErr.(*DownloadDirPartialError) - // Clean up any remaining temporary directories from previous runs. + // Clean up any remaining temporary directories from previous runs, as well + // as partially extracted diectories created by future versions of cmd/go. // This is only safe to do because the lock file ensures that their writers // are no longer active. parentDir := filepath.Dir(dir) @@ -95,6 +101,19 @@ func download(mod module.Version, dir string) (err error) { RemoveAll(path) // best effort } } + if dirExists { + if err := RemoveAll(dir); err != nil { + return "", err + } + } + + partialPath, err := CachePath(mod, "partial") + if err != nil { + return "", err + } + if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) { + return "", err + } // Extract the zip file to a temporary directory, then rename it to the // final path. That way, we can use the existence of the source directory to @@ -102,11 +121,11 @@ func download(mod module.Version, dir string) (err error) { // the entire directory (e.g. as an attempt to prune out file corruption) // the module cache will still be left in a recoverable state. if err := os.MkdirAll(parentDir, 0777); err != nil { - return err + return "", err } tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix) if err != nil { - return err + return "", err } defer func() { if err != nil { @@ -116,11 +135,11 @@ func download(mod module.Version, dir string) (err error) { if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil { fmt.Fprintf(os.Stderr, "-> %s\n", err) - return err + return "", err } - if err := os.Rename(tmpDir, dir); err != nil { - return err + if err := robustio.Rename(tmpDir, dir); err != nil { + return "", err } if !cfg.ModCacheRW { @@ -128,7 +147,7 @@ func download(mod module.Version, dir string) (err error) { // os.Rename was observed to fail for read-only directories on macOS. makeDirsReadOnly(dir) } - return nil + return dir, nil } var downloadZipCache par.Cache diff --git a/libgo/go/cmd/go/internal/modload/build.go b/libgo/go/cmd/go/internal/modload/build.go index d0642bc..2a69d59 100644 --- a/libgo/go/cmd/go/internal/modload/build.go +++ b/libgo/go/cmd/go/internal/modload/build.go @@ -148,9 +148,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { } dir, err := modfetch.DownloadDir(mod) if err == nil { - if info, err := os.Stat(dir); err == nil && info.IsDir() { - m.Dir = dir - } + m.Dir = dir } } } diff --git a/libgo/go/cmd/go/internal/modload/help.go b/libgo/go/cmd/go/internal/modload/help.go index 66c1f70..bd19bb4 100644 --- a/libgo/go/cmd/go/internal/modload/help.go +++ b/libgo/go/cmd/go/internal/modload/help.go @@ -176,13 +176,21 @@ not need updates, such as in a continuous integration and testing system. The "go get" command remains permitted to update go.mod even with -mod=readonly, and the "go mod" commands do not take the -mod flag (or any other build flags). -If invoked with -mod=vendor, the go command assumes that the vendor -directory holds the correct copies of dependencies and ignores -the dependency descriptions in go.mod. +If invoked with -mod=vendor, the go command loads packages from the main +module's vendor directory instead of downloading modules to and loading packages +from the module cache. The go command assumes the vendor directory holds +correct copies of dependencies, and it does not compute the set of required +module versions from go.mod files. However, the go command does check that +vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent +with go.mod. If invoked with -mod=mod, the go command loads modules from the module cache even if there is a vendor directory present. +If the go command is not invoked with a -mod flag and the vendor directory +is present and the "go" version in go.mod is 1.14 or higher, the go command +will act as if it were invoked with -mod=vendor. + Pseudo-versions The go.mod file and the go command more generally use semantic versions as @@ -380,22 +388,28 @@ the format of the cached downloaded packages. Modules and vendoring -When using modules, the go command completely ignores vendor directories. - -By default, the go command satisfies dependencies by downloading modules -from their sources and using those downloaded copies (after verification, -as described in the previous section). To allow interoperation with older -versions of Go, or to ensure that all files used for a build are stored -together in a single file tree, 'go mod vendor' creates a directory named -vendor in the root directory of the main module and stores there all the -packages from dependency modules that are needed to support builds and -tests of packages in the main module. - -To build using the main module's top-level vendor directory to satisfy -dependencies (disabling use of the usual network sources and local -caches), use 'go build -mod=vendor'. Note that only the main module's -top-level vendor directory is used; vendor directories in other locations -are still ignored. +When using modules, the go command typically satisfies dependencies by +downloading modules from their sources and using those downloaded copies +(after verification, as described in the previous section). Vendoring may +be used to allow interoperation with older versions of Go, or to ensure +that all files used for a build are stored together in a single file tree. + +The command 'go mod vendor' constructs a directory named vendor in the main +module's root directory that contains copies of all packages needed to support +builds and tests of packages in the main module. 'go mod vendor' also +creates the file vendor/modules.txt that contains metadata about vendored +packages and module versions. This file should be kept consistent with go.mod: +when vendoring is used, 'go mod vendor' should be run after go.mod is updated. + +If the vendor directory is present in the main module's root directory, it will +be used automatically if the "go" version in the main module's go.mod file is +1.14 or higher. Build commands like 'go build' and 'go test' will load packages +from the vendor directory instead of accessing the network or the local module +cache. To explicitly enable vendoring, invoke the go command with the flag +-mod=vendor. To disable vendoring, use the flag -mod=mod. + +Unlike vendoring in GOPATH, the go command ignores vendor directories in +locations other than the main module's root directory. `, } diff --git a/libgo/go/cmd/go/internal/robustio/robustio_flaky.go b/libgo/go/cmd/go/internal/robustio/robustio_flaky.go index e57c8c7..d4cb7e6 100644 --- a/libgo/go/cmd/go/internal/robustio/robustio_flaky.go +++ b/libgo/go/cmd/go/internal/robustio/robustio_flaky.go @@ -15,7 +15,7 @@ import ( "time" ) -const arbitraryTimeout = 500 * time.Millisecond +const arbitraryTimeout = 2000 * time.Millisecond // retry retries ephemeral errors from f up to an arbitrary timeout // to work around filesystem flakiness on Windows and Darwin. diff --git a/libgo/go/runtime/mbitmap.go b/libgo/go/runtime/mbitmap.go index 457da13..be8e00c 100644 --- a/libgo/go/runtime/mbitmap.go +++ b/libgo/go/runtime/mbitmap.go @@ -1941,7 +1941,11 @@ Run: // The bitmask starts at s.startAddr. // The result must be deallocated with dematerializeGCProg. func materializeGCProg(ptrdata uintptr, prog *byte) *mspan { - s := mheap_.allocManual((ptrdata/(8*sys.PtrSize)+pageSize-1)/pageSize, &memstats.gc_sys) + // Each word of ptrdata needs one bit in the bitmap. + bitmapBytes := divRoundUp(ptrdata, 8*sys.PtrSize) + // Compute the number of pages needed for bitmapBytes. + pages := divRoundUp(bitmapBytes, pageSize) + s := mheap_.allocManual(pages, &memstats.gc_sys) runGCProg(addb(prog, 4), nil, (*byte)(unsafe.Pointer(s.startAddr)), 1) return s } diff --git a/libgo/go/runtime/stubs.go b/libgo/go/runtime/stubs.go index ae6134d..4a06da5 100644 --- a/libgo/go/runtime/stubs.go +++ b/libgo/go/runtime/stubs.go @@ -250,6 +250,13 @@ func alignDown(n, a uintptr) uintptr { return n &^ (a - 1) } +// divRoundUp returns ceil(n / a). +func divRoundUp(n, a uintptr) uintptr { + // a is generally a power of two. This will get inlined and + // the compiler will optimize the division. + return (n + a - 1) / a +} + // checkASM returns whether assembly runtime checks have passed. func checkASM() bool { return true -- cgit v1.1