diff options
Diffstat (limited to 'libgo/go/cmd')
46 files changed, 2273 insertions, 737 deletions
diff --git a/libgo/go/cmd/cgo/ast.go b/libgo/go/cmd/cgo/ast.go index 8bbd1cc..c3a24c2 100644 --- a/libgo/go/cmd/cgo/ast.go +++ b/libgo/go/cmd/cgo/ast.go @@ -124,7 +124,7 @@ func (f *File) ReadGo(name string) { if f.Ref == nil { f.Ref = make([]*Ref, 0, 8) } - f.walk(ast2, "prog", (*File).saveRef) + f.walk(ast2, "prog", (*File).saveExprs) // Accumulate exported functions. // The comments are only on ast1 but we need to @@ -163,52 +163,72 @@ func commentText(g *ast.CommentGroup) string { return strings.Join(pieces, "") } +// Save various references we are going to need later. +func (f *File) saveExprs(x interface{}, context string) { + switch x := x.(type) { + case *ast.Expr: + switch (*x).(type) { + case *ast.SelectorExpr: + f.saveRef(x, context) + } + case *ast.CallExpr: + f.saveCall(x) + } +} + // Save references to C.xxx for later processing. -func (f *File) saveRef(x interface{}, context string) { - n, ok := x.(*ast.Expr) - if !ok { +func (f *File) saveRef(n *ast.Expr, context string) { + sel := (*n).(*ast.SelectorExpr) + // For now, assume that the only instance of capital C is when + // used as the imported package identifier. + // The parser should take care of scoping in the future, so + // that we will be able to distinguish a "top-level C" from a + // local C. + if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" { return } - if sel, ok := (*n).(*ast.SelectorExpr); ok { - // For now, assume that the only instance of capital C is - // when used as the imported package identifier. - // The parser should take care of scoping in the future, - // so that we will be able to distinguish a "top-level C" - // from a local C. - if l, ok := sel.X.(*ast.Ident); ok && l.Name == "C" { - if context == "as2" { - context = "expr" - } - if context == "embed-type" { - error_(sel.Pos(), "cannot embed C type") - } - goname := sel.Sel.Name - if goname == "errno" { - error_(sel.Pos(), "cannot refer to errno directly; see documentation") - return - } - if goname == "_CMalloc" { - error_(sel.Pos(), "cannot refer to C._CMalloc; use C.malloc") - return - } - if goname == "malloc" { - goname = "_CMalloc" - } - name := f.Name[goname] - if name == nil { - name = &Name{ - Go: goname, - } - f.Name[goname] = name - } - f.Ref = append(f.Ref, &Ref{ - Name: name, - Expr: n, - Context: context, - }) - return + if context == "as2" { + context = "expr" + } + if context == "embed-type" { + error_(sel.Pos(), "cannot embed C type") + } + goname := sel.Sel.Name + if goname == "errno" { + error_(sel.Pos(), "cannot refer to errno directly; see documentation") + return + } + if goname == "_CMalloc" { + error_(sel.Pos(), "cannot refer to C._CMalloc; use C.malloc") + return + } + if goname == "malloc" { + goname = "_CMalloc" + } + name := f.Name[goname] + if name == nil { + name = &Name{ + Go: goname, } + f.Name[goname] = name + } + f.Ref = append(f.Ref, &Ref{ + Name: name, + Expr: n, + Context: context, + }) +} + +// Save calls to C.xxx for later processing. +func (f *File) saveCall(call *ast.CallExpr) { + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return + } + if l, ok := sel.X.(*ast.Ident); !ok || l.Name != "C" { + return } + f.Calls = append(f.Calls, call) } // If a function should be exported add it to ExpFunc. diff --git a/libgo/go/cmd/cgo/doc.go b/libgo/go/cmd/cgo/doc.go index b2a5428..bd38a5c 100644 --- a/libgo/go/cmd/cgo/doc.go +++ b/libgo/go/cmd/cgo/doc.go @@ -117,17 +117,27 @@ The standard C numeric types are available under the names C.char, C.schar (signed char), C.uchar (unsigned char), C.short, C.ushort (unsigned short), C.int, C.uint (unsigned int), C.long, C.ulong (unsigned long), C.longlong (long long), -C.ulonglong (unsigned long long), C.float, C.double. +C.ulonglong (unsigned long long), C.float, C.double, +C.complexfloat (complex float), and C.complexdouble (complex double). The C type void* is represented by Go's unsafe.Pointer. +The C types __int128_t and __uint128_t are represented by [16]byte. To access a struct, union, or enum type directly, prefix it with struct_, union_, or enum_, as in C.struct_stat. +The size of any C type T is available as C.sizeof_T, as in +C.sizeof_struct_stat. + As Go doesn't have support for C's union type in the general case, C's union types are represented as a Go byte array with the same length. Go structs cannot embed fields with C types. +Go code can not refer to zero-sized fields that occur at the end of +non-empty C structs. To get the address of such a field (which is the +only operation you can do with a zero-sized field) you must take the +address of the struct and add the size of the struct. + Cgo translates C types into equivalent unexported Go types. Because the translations are unexported, a Go package should not expose C types in its exported API: a C type used in one Go package @@ -188,10 +198,10 @@ by making copies of the data. In pseudo-Go definitions: // C string to Go string func C.GoString(*C.char) string - // C string, length to Go string + // C data with explicit length to Go string func C.GoStringN(*C.char, C.int) string - // C pointer, length to Go []byte + // C data with explicit length to Go []byte func C.GoBytes(unsafe.Pointer, C.int) []byte C references to Go @@ -221,6 +231,55 @@ definitions and declarations, then the two output files will produce duplicate symbols and the linker will fail. To avoid this, definitions must be placed in preambles in other files, or in C source files. +Passing pointers + +Go is a garbage collected language, and the garbage collector needs to +know the location of every pointer to Go memory. Because of this, +there are restrictions on passing pointers between Go and C. + +In this section the term Go pointer means a pointer to memory +allocated by Go (such as by using the & operator or calling the +predefined new function) and the term C pointer means a pointer to +memory allocated by C (such as by a call to C.malloc). Whether a +pointer is a Go pointer or a C pointer is a dynamic property +determined by how the memory was allocated; it has nothing to do with +the type of the pointer. + +Go code may pass a Go pointer to C provided the Go memory to which it +points does not contain any Go pointers. The C code must preserve +this property: it must not store any Go pointers in Go memory, even +temporarily. When passing a pointer to a field in a struct, the Go +memory in question is the memory occupied by the field, not the entire +struct. When passing a pointer to an element in an array or slice, +the Go memory in question is the entire array or the entire backing +array of the slice. + +C code may not keep a copy of a Go pointer after the call returns. + +A Go function called by C code may not return a Go pointer. A Go +function called by C code may take C pointers as arguments, and it may +store non-pointer or C pointer data through those pointers, but it may +not store a Go pointer in memory pointed to by a C pointer. A Go +function called by C code may take a Go pointer as an argument, but it +must preserve the property that the Go memory to which it points does +not contain any Go pointers. + +Go code may not store a Go pointer in C memory. C code may store Go +pointers in C memory, subject to the rule above: it must stop storing +the Go pointer when the C function returns. + +These rules are checked dynamically at runtime. The checking is +controlled by the cgocheck setting of the GODEBUG environment +variable. The default setting is GODEBUG=cgocheck=1, which implements +reasonably cheap dynamic checks. These checks may be disabled +entirely using GODEBUG=cgocheck=0. Complete checking of pointer +handling, at some cost in run time, is available via GODEBUG=cgocheck=2. + +It is possible to defeat this enforcement by using the unsafe package, +and of course there is nothing stopping the C code from doing anything +it likes. However, programs that break these rules are likely to fail +in unexpected and unpredictable ways. + Using cgo directly Usage: @@ -391,17 +450,13 @@ the translation process. Translating Go -[The rest of this comment refers to 6g, the Go compiler that is part -of the amd64 port of the gc Go toolchain. Everything here applies to -another architecture's compilers as well.] - Given the input Go files x.go and y.go, cgo generates these source files: - x.cgo1.go # for 6g - y.cgo1.go # for 6g - _cgo_gotypes.go # for 6g - _cgo_import.go # for 6g (if -dynout _cgo_import.go) + x.cgo1.go # for gc (cmd/compile) + y.cgo1.go # for gc + _cgo_gotypes.go # for gc + _cgo_import.go # for gc (if -dynout _cgo_import.go) x.cgo2.c # for gcc y.cgo2.c # for gcc _cgo_defun.c # for gcc (if -gccgo) @@ -464,7 +519,7 @@ Linking Once the _cgo_export.c and *.cgo2.c files have been compiled with gcc, they need to be linked into the final binary, along with the libraries -they might depend on (in the case of puts, stdio). 6l has been +they might depend on (in the case of puts, stdio). cmd/link has been extended to understand basic ELF files, but it does not understand ELF in the full complexity that modern C libraries embrace, so it cannot in general generate direct references to the system libraries. @@ -495,23 +550,23 @@ _cgo_import.go, which looks like: //go:cgo_import_dynamic _ _ "libc.so.6" In the end, the compiled Go package, which will eventually be -presented to 6l as part of a larger program, contains: +presented to cmd/link as part of a larger program, contains: - _go_.6 # 6g-compiled object for _cgo_gotypes.go, _cgo_import.go, *.cgo1.go + _go_.o # gc-compiled object for _cgo_gotypes.go, _cgo_import.go, *.cgo1.go _all.o # gcc-compiled object for _cgo_export.c, *.cgo2.c -The final program will be a dynamic executable, so that 6l can avoid +The final program will be a dynamic executable, so that cmd/link can avoid needing to process arbitrary .o files. It only needs to process the .o files generated from C files that cgo writes, and those are much more limited in the ELF or other features that they use. -In essence, the _cgo_import.6 file includes the extra linking -directives that 6l is not sophisticated enough to derive from _all.o +In essence, the _cgo_import.o file includes the extra linking +directives that cmd/link is not sophisticated enough to derive from _all.o on its own. Similarly, the _all.o uses dynamic references to real -system object code because 6l is not sophisticated enough to process +system object code because cmd/link is not sophisticated enough to process the real code. -The main benefits of this system are that 6l remains relatively simple +The main benefits of this system are that cmd/link remains relatively simple (it does not need to implement a complete ELF and Mach-O linker) and that gcc is not needed after the package is compiled. For example, package net uses cgo for access to name resolution functions provided @@ -540,17 +595,17 @@ system calls. Internal and External Linking -The text above describes "internal" linking, in which 6l parses and +The text above describes "internal" linking, in which cmd/link parses and links host object files (ELF, Mach-O, PE, and so on) into the final -executable itself. Keeping 6l simple means we cannot possibly +executable itself. Keeping cmd/link simple means we cannot possibly implement the full semantics of the host linker, so the kinds of objects that can be linked directly into the binary is limited (other code can only be used as a dynamic library). On the other hand, when -using internal linking, 6l can generate Go binaries by itself. +using internal linking, cmd/link can generate Go binaries by itself. In order to allow linking arbitrary object files without requiring dynamic libraries, cgo supports an "external" linking mode too. In -external linking mode, 6l does not process any host object files. +external linking mode, cmd/link does not process any host object files. Instead, it collects all the Go code and writes a single go.o object file containing it. Then it invokes the host linker (usually gcc) to combine the go.o object file and any supporting non-Go code into a @@ -582,8 +637,8 @@ to be made when linking the final binary. Linking Directives In either linking mode, package-specific directives must be passed -through to 6l. These are communicated by writing //go: directives in a -Go source file compiled by 6g. The directives are copied into the .6 +through to cmd/link. These are communicated by writing //go: directives in a +Go source file compiled by gc. The directives are copied into the .o object file and then processed by the linker. The directives are: @@ -672,7 +727,7 @@ Example As a simple example, consider a package that uses cgo to call C.sin. The following code will be generated by cgo: - // compiled by 6g + // compiled by gc //go:cgo_ldflag "-lm" @@ -708,7 +763,7 @@ Otherwise the link will be an internal one. The linking directives are used according to the kind of final link used. -In internal mode, 6l itself processes all the host object files, in +In internal mode, cmd/link itself processes all the host object files, in particular foo.cgo2.o. To do so, it uses the cgo_import_dynamic and cgo_dynamic_linker directives to learn that the otherwise undefined reference to sin in foo.cgo2.o should be rewritten to refer to the @@ -716,56 +771,56 @@ symbol sin with version GLIBC_2.2.5 from the dynamic library "libm.so.6", and the binary should request "/lib/ld-linux.so.2" as its runtime dynamic linker. -In external mode, 6l does not process any host object files, in -particular foo.cgo2.o. It links together the 6g-generated object +In external mode, cmd/link does not process any host object files, in +particular foo.cgo2.o. It links together the gc-generated object files, along with any other Go code, into a go.o file. While doing -that, 6l will discover that there is no definition for -_cgo_gcc_Cfunc_sin, referred to by the 6g-compiled source file. This -is okay, because 6l also processes the cgo_import_static directive and +that, cmd/link will discover that there is no definition for +_cgo_gcc_Cfunc_sin, referred to by the gc-compiled source file. This +is okay, because cmd/link also processes the cgo_import_static directive and knows that _cgo_gcc_Cfunc_sin is expected to be supplied by a host -object file, so 6l does not treat the missing symbol as an error when +object file, so cmd/link does not treat the missing symbol as an error when creating go.o. Indeed, the definition for _cgo_gcc_Cfunc_sin will be provided to the host linker by foo2.cgo.o, which in turn will need the -symbol 'sin'. 6l also processes the cgo_ldflag directives, so that it +symbol 'sin'. cmd/link also processes the cgo_ldflag directives, so that it knows that the eventual host link command must include the -lm argument, so that the host linker will be able to find 'sin' in the math library. -6l Command Line Interface +cmd/link Command Line Interface -The go command and any other Go-aware build systems invoke 6l -to link a collection of packages into a single binary. By default, 6l will +The go command and any other Go-aware build systems invoke cmd/link +to link a collection of packages into a single binary. By default, cmd/link will present the same interface it does today: - 6l main.a + cmd/link main.a -produces a file named 6.out, even if 6l does so by invoking the host +produces a file named a.out, even if cmd/link does so by invoking the host linker in external linking mode. -By default, 6l will decide the linking mode as follows: if the only +By default, cmd/link will decide the linking mode as follows: if the only packages using cgo are those on a whitelist of standard library -packages (net, os/user, runtime/cgo), 6l will use internal linking -mode. Otherwise, there are non-standard cgo packages involved, and 6l +packages (net, os/user, runtime/cgo), cmd/link will use internal linking +mode. Otherwise, there are non-standard cgo packages involved, and cmd/link will use external linking mode. The first rule means that a build of the godoc binary, which uses net but no other cgo, can run without needing gcc available. The second rule means that a build of a cgo-wrapped library like sqlite3 can generate a standalone executable instead of needing to refer to a dynamic library. The specific choice -can be overridden using a command line flag: 6l -linkmode=internal or -6l -linkmode=external. +can be overridden using a command line flag: cmd/link -linkmode=internal or +cmd/link -linkmode=external. -In an external link, 6l will create a temporary directory, write any +In an external link, cmd/link will create a temporary directory, write any host object files found in package archives to that directory (renamed to avoid conflicts), write the go.o file to that directory, and invoke the host linker. The default value for the host linker is $CC, split into fields, or else "gcc". The specific host linker command line can -be overridden using command line flags: 6l -extld=clang +be overridden using command line flags: cmd/link -extld=clang -extldflags='-ggdb -O3'. If any package in a build includes a .cc or other file compiled by the C++ compiler, the go tool will use the -extld option to set the host linker to the C++ compiler. These defaults mean that Go-aware build systems can ignore the linking -changes and keep running plain '6l' and get reasonable results, but +changes and keep running plain 'cmd/link' and get reasonable results, but they can also control the linking details if desired. */ diff --git a/libgo/go/cmd/cgo/gcc.go b/libgo/go/cmd/cgo/gcc.go index e0b89ec..fb5049c 100644 --- a/libgo/go/cmd/cgo/gcc.go +++ b/libgo/go/cmd/cgo/gcc.go @@ -38,8 +38,8 @@ var nameToC = map[string]string{ "ulong": "unsigned long", "longlong": "long long", "ulonglong": "unsigned long long", - "complexfloat": "float complex", - "complexdouble": "double complex", + "complexfloat": "float _Complex", + "complexdouble": "double _Complex", } // cname returns the C name to use for C.s. @@ -167,6 +167,7 @@ func (p *Package) Translate(f *File) { if len(needType) > 0 { p.loadDWARF(f, needType) } + p.rewriteCalls(f) p.rewriteRef(f) } @@ -575,6 +576,331 @@ func (p *Package) mangleName(n *Name) { n.Mangle = prefix + n.Kind + "_" + n.Go } +// rewriteCalls rewrites all calls that pass pointers to check that +// they follow the rules for passing pointers between Go and C. +func (p *Package) rewriteCalls(f *File) { + for _, call := range f.Calls { + // This is a call to C.xxx; set goname to "xxx". + goname := call.Fun.(*ast.SelectorExpr).Sel.Name + if goname == "malloc" { + continue + } + name := f.Name[goname] + if name.Kind != "func" { + // Probably a type conversion. + continue + } + p.rewriteCall(f, call, name) + } +} + +// rewriteCall rewrites one call to add pointer checks. We replace +// each pointer argument x with _cgoCheckPointer(x).(T). +func (p *Package) rewriteCall(f *File, call *ast.CallExpr, name *Name) { + for i, param := range name.FuncType.Params { + if len(call.Args) <= i { + // Avoid a crash; this will be caught when the + // generated file is compiled. + return + } + + // An untyped nil does not need a pointer check, and + // when _cgoCheckPointer returns the untyped nil the + // type assertion we are going to insert will fail. + // Easier to just skip nil arguments. + // TODO: Note that this fails if nil is shadowed. + if id, ok := call.Args[i].(*ast.Ident); ok && id.Name == "nil" { + continue + } + + if !p.needsPointerCheck(f, param.Go) { + continue + } + + c := &ast.CallExpr{ + Fun: ast.NewIdent("_cgoCheckPointer"), + Args: []ast.Expr{ + call.Args[i], + }, + } + + // Add optional additional arguments for an address + // expression. + c.Args = p.checkAddrArgs(f, c.Args, call.Args[i]) + + // _cgoCheckPointer returns interface{}. + // We need to type assert that to the type we want. + // If the Go version of this C type uses + // unsafe.Pointer, we can't use a type assertion, + // because the Go file might not import unsafe. + // Instead we use a local variant of _cgoCheckPointer. + + var arg ast.Expr + if n := p.unsafeCheckPointerName(param.Go); n != "" { + c.Fun = ast.NewIdent(n) + arg = c + } else { + // In order for the type assertion to succeed, + // we need it to match the actual type of the + // argument. The only type we have is the + // type of the function parameter. We know + // that the argument type must be assignable + // to the function parameter type, or the code + // would not compile, but there is nothing + // requiring that the types be exactly the + // same. Add a type conversion to the + // argument so that the type assertion will + // succeed. + c.Args[0] = &ast.CallExpr{ + Fun: param.Go, + Args: []ast.Expr{ + c.Args[0], + }, + } + + arg = &ast.TypeAssertExpr{ + X: c, + Type: param.Go, + } + } + + call.Args[i] = arg + } +} + +// needsPointerCheck returns whether the type t needs a pointer check. +// This is true if t is a pointer and if the value to which it points +// might contain a pointer. +func (p *Package) needsPointerCheck(f *File, t ast.Expr) bool { + return p.hasPointer(f, t, true) +} + +// hasPointer is used by needsPointerCheck. If top is true it returns +// whether t is or contains a pointer that might point to a pointer. +// If top is false it returns whether t is or contains a pointer. +// f may be nil. +func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool { + switch t := t.(type) { + case *ast.ArrayType: + if t.Len == nil { + if !top { + return true + } + return p.hasPointer(f, t.Elt, false) + } + return p.hasPointer(f, t.Elt, top) + case *ast.StructType: + for _, field := range t.Fields.List { + if p.hasPointer(f, field.Type, top) { + return true + } + } + return false + case *ast.StarExpr: // Pointer type. + if !top { + return true + } + return p.hasPointer(f, t.X, false) + case *ast.FuncType, *ast.InterfaceType, *ast.MapType, *ast.ChanType: + return true + case *ast.Ident: + // TODO: Handle types defined within function. + for _, d := range p.Decl { + gd, ok := d.(*ast.GenDecl) + if !ok || gd.Tok != token.TYPE { + continue + } + for _, spec := range gd.Specs { + ts, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + if ts.Name.Name == t.Name { + return p.hasPointer(f, ts.Type, top) + } + } + } + if def := typedef[t.Name]; def != nil { + return p.hasPointer(f, def.Go, top) + } + if t.Name == "string" { + return !top + } + if t.Name == "error" { + return true + } + if goTypes[t.Name] != nil { + return false + } + // We can't figure out the type. Conservative + // approach is to assume it has a pointer. + return true + case *ast.SelectorExpr: + if l, ok := t.X.(*ast.Ident); !ok || l.Name != "C" { + // Type defined in a different package. + // Conservative approach is to assume it has a + // pointer. + return true + } + if f == nil { + // Conservative approach: assume pointer. + return true + } + name := f.Name[t.Sel.Name] + if name != nil && name.Kind == "type" && name.Type != nil && name.Type.Go != nil { + return p.hasPointer(f, name.Type.Go, top) + } + // We can't figure out the type. Conservative + // approach is to assume it has a pointer. + return true + default: + error_(t.Pos(), "could not understand type %s", gofmt(t)) + return true + } +} + +// checkAddrArgs tries to add arguments to the call of +// _cgoCheckPointer when the argument is an address expression. We +// pass true to mean that the argument is an address operation of +// something other than a slice index, which means that it's only +// necessary to check the specific element pointed to, not the entire +// object. This is for &s.f, where f is a field in a struct. We can +// pass a slice or array, meaning that we should check the entire +// slice or array but need not check any other part of the object. +// This is for &s.a[i], where we need to check all of a. However, we +// only pass the slice or array if we can refer to it without side +// effects. +func (p *Package) checkAddrArgs(f *File, args []ast.Expr, x ast.Expr) []ast.Expr { + // Strip type conversions. + for { + c, ok := x.(*ast.CallExpr) + if !ok || len(c.Args) != 1 || !p.isType(c.Fun) { + break + } + x = c.Args[0] + } + u, ok := x.(*ast.UnaryExpr) + if !ok || u.Op != token.AND { + return args + } + index, ok := u.X.(*ast.IndexExpr) + if !ok { + // This is the address of something that is not an + // index expression. We only need to examine the + // single value to which it points. + // TODO: what if true is shadowed? + return append(args, ast.NewIdent("true")) + } + if !p.hasSideEffects(f, index.X) { + // Examine the entire slice. + return append(args, index.X) + } + // Treat the pointer as unknown. + return args +} + +// hasSideEffects returns whether the expression x has any side +// effects. x is an expression, not a statement, so the only side +// effect is a function call. +func (p *Package) hasSideEffects(f *File, x ast.Expr) bool { + found := false + f.walk(x, "expr", + func(f *File, x interface{}, context string) { + switch x.(type) { + case *ast.CallExpr: + found = true + } + }) + return found +} + +// isType returns whether the expression is definitely a type. +// This is conservative--it returns false for an unknown identifier. +func (p *Package) isType(t ast.Expr) bool { + switch t := t.(type) { + case *ast.SelectorExpr: + if t.Sel.Name != "Pointer" { + return false + } + id, ok := t.X.(*ast.Ident) + if !ok { + return false + } + return id.Name == "unsafe" + case *ast.Ident: + // TODO: This ignores shadowing. + switch t.Name { + case "unsafe.Pointer", "bool", "byte", + "complex64", "complex128", + "error", + "float32", "float64", + "int", "int8", "int16", "int32", "int64", + "rune", "string", + "uint", "uint8", "uint16", "uint32", "uint64", "uintptr": + + return true + } + case *ast.StarExpr: + return p.isType(t.X) + case *ast.ArrayType, *ast.StructType, *ast.FuncType, *ast.InterfaceType, + *ast.MapType, *ast.ChanType: + + return true + } + return false +} + +// unsafeCheckPointerName is given the Go version of a C type. If the +// type uses unsafe.Pointer, we arrange to build a version of +// _cgoCheckPointer that returns that type. This avoids using a type +// assertion to unsafe.Pointer in our copy of user code. We return +// the name of the _cgoCheckPointer function we are going to build, or +// the empty string if the type does not use unsafe.Pointer. +func (p *Package) unsafeCheckPointerName(t ast.Expr) string { + if !p.hasUnsafePointer(t) { + return "" + } + var buf bytes.Buffer + conf.Fprint(&buf, fset, t) + s := buf.String() + for i, t := range p.CgoChecks { + if s == t { + return p.unsafeCheckPointerNameIndex(i) + } + } + p.CgoChecks = append(p.CgoChecks, s) + return p.unsafeCheckPointerNameIndex(len(p.CgoChecks) - 1) +} + +// hasUnsafePointer returns whether the Go type t uses unsafe.Pointer. +// t is the Go version of a C type, so we don't need to handle every case. +// We only care about direct references, not references via typedefs. +func (p *Package) hasUnsafePointer(t ast.Expr) bool { + switch t := t.(type) { + case *ast.Ident: + // We don't see a SelectorExpr for unsafe.Pointer; + // this is created by code in this file. + return t.Name == "unsafe.Pointer" + case *ast.ArrayType: + return p.hasUnsafePointer(t.Elt) + case *ast.StructType: + for _, f := range t.Fields.List { + if p.hasUnsafePointer(f.Type) { + return true + } + } + case *ast.StarExpr: // Pointer type. + return p.hasUnsafePointer(t.X) + } + return false +} + +// unsafeCheckPointerNameIndex returns the name to use for a +// _cgoCheckPointer variant based on the index in the CgoChecks slice. +func (p *Package) unsafeCheckPointerNameIndex(i int) string { + return fmt.Sprintf("_cgoCheckPointer%d", i) +} + // rewriteRef rewrites all the C.xxx references in f.AST to refer to the // Go equivalents, now that we have figured out the meaning of all // the xxx. In *godefs mode, rewriteRef replaces the names @@ -612,6 +938,10 @@ func (p *Package) rewriteRef(f *File) { if r.Name.Kind != "func" { if r.Name.Kind == "type" { r.Context = "type" + if r.Name.Type == nil { + error_(r.Pos(), "invalid conversion to C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C) + break + } expr = r.Name.Type.Go break } @@ -663,6 +993,10 @@ func (p *Package) rewriteRef(f *File) { } } else if r.Name.Kind == "type" { // Okay - might be new(T) + if r.Name.Type == nil { + error_(r.Pos(), "expression C.%s: undefined C type '%s'", fixGo(r.Name.Go), r.Name.C) + break + } expr = r.Name.Type.Go } else if r.Name.Kind == "var" { expr = &ast.StarExpr{Star: (*r.Expr).Pos(), X: expr} @@ -1028,12 +1362,12 @@ var dwarfToName = map[string]string{ "long unsigned int": "ulong", "unsigned int": "uint", "short unsigned int": "ushort", + "unsigned short": "ushort", // Used by Clang; issue 13129. "short int": "short", "long long int": "longlong", "long long unsigned int": "ulonglong", "signed char": "schar", - "float complex": "complexfloat", - "double complex": "complexdouble", + "unsigned char": "uchar", } const signedDelta = 64 @@ -1224,6 +1558,11 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type { t.Go = c.int32 case 8: t.Go = c.int64 + case 16: + t.Go = &ast.ArrayType{ + Len: c.intExpr(t.Size), + Elt: c.uint8, + } } if t.Align = t.Size; t.Align >= c.ptrSize { t.Align = c.ptrSize @@ -1381,6 +1720,11 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type { t.Go = c.uint32 case 8: t.Go = c.uint64 + case 16: + t.Go = &ast.ArrayType{ + Len: c.intExpr(t.Size), + Elt: c.uint8, + } } if t.Align = t.Size; t.Align >= c.ptrSize { t.Align = c.ptrSize @@ -1393,7 +1737,7 @@ func (c *typeConv) Type(dtype dwarf.Type, pos token.Pos) *Type { } switch dtype.(type) { - case *dwarf.AddrType, *dwarf.BoolType, *dwarf.CharType, *dwarf.IntType, *dwarf.FloatType, *dwarf.UcharType, *dwarf.UintType: + case *dwarf.AddrType, *dwarf.BoolType, *dwarf.CharType, *dwarf.ComplexType, *dwarf.IntType, *dwarf.FloatType, *dwarf.UcharType, *dwarf.UintType: s := dtype.Common().Name if s != "" { if ss, ok := dwarfToName[s]; ok { diff --git a/libgo/go/cmd/cgo/godefs.go b/libgo/go/cmd/cgo/godefs.go index 1b0ece2..aff616e 100644 --- a/libgo/go/cmd/cgo/godefs.go +++ b/libgo/go/cmd/cgo/godefs.go @@ -11,6 +11,7 @@ import ( "go/printer" "go/token" "os" + "path/filepath" "strings" ) @@ -19,7 +20,7 @@ func (p *Package) godefs(f *File, srcfile string) string { var buf bytes.Buffer fmt.Fprintf(&buf, "// Created by cgo -godefs - DO NOT EDIT\n") - fmt.Fprintf(&buf, "// %s\n", strings.Join(os.Args, " ")) + fmt.Fprintf(&buf, "// %s %s\n", filepath.Base(os.Args[0]), strings.Join(os.Args[1:], " ")) fmt.Fprintf(&buf, "\n") override := make(map[string]string) diff --git a/libgo/go/cmd/cgo/main.go b/libgo/go/cmd/cgo/main.go index c8cd161..6171b9d 100644 --- a/libgo/go/cmd/cgo/main.go +++ b/libgo/go/cmd/cgo/main.go @@ -42,6 +42,7 @@ type Package struct { GoFiles []string // list of Go files GccFiles []string // list of gcc output files Preamble string // collected preamble for _cgo_export.h + CgoChecks []string // see unsafeCheckPointerName } // A File collects information about a single Go input file. @@ -51,6 +52,7 @@ type File struct { Package string // Package name Preamble string // C preamble (doc comment on import "C") Ref []*Ref // all references to C.xxx in AST + Calls []*ast.CallExpr // all calls to C.xxx in AST ExpFunc []*ExpFunc // exported functions for this file Name map[string]*Name // map from Go name to Name } @@ -132,43 +134,47 @@ func usage() { } var ptrSizeMap = map[string]int64{ - "386": 4, - "alpha": 8, - "amd64": 8, - "arm": 4, - "arm64": 8, - "m68k": 4, - "mipso32": 4, - "mipsn32": 4, - "mipso64": 8, - "mipsn64": 8, - "ppc": 4, - "ppc64": 8, - "ppc64le": 8, - "s390": 4, - "s390x": 8, - "sparc": 4, - "sparc64": 8, + "386": 4, + "alpha": 8, + "amd64": 8, + "arm": 4, + "arm64": 8, + "m68k": 4, + "mipso32": 4, + "mipsn32": 4, + "mipso64": 8, + "mipsn64": 8, + "mips64": 8, + "mips64le": 8, + "ppc": 4, + "ppc64": 8, + "ppc64le": 8, + "s390": 4, + "s390x": 8, + "sparc": 4, + "sparc64": 8, } var intSizeMap = map[string]int64{ - "386": 4, - "alpha": 8, - "amd64": 8, - "arm": 4, - "arm64": 8, - "m68k": 4, - "mipso32": 4, - "mipsn32": 4, - "mipso64": 8, - "mipsn64": 8, - "ppc": 4, - "ppc64": 8, - "ppc64le": 8, - "s390": 4, - "s390x": 8, - "sparc": 4, - "sparc64": 8, + "386": 4, + "alpha": 8, + "amd64": 8, + "arm": 4, + "arm64": 8, + "m68k": 4, + "mipso32": 4, + "mipsn32": 4, + "mipso64": 8, + "mipsn64": 8, + "mips64": 8, + "mips64le": 8, + "ppc": 4, + "ppc64": 8, + "ppc64le": 8, + "s390": 4, + "s390x": 8, + "sparc": 4, + "sparc64": 8, } var cPrefix string @@ -297,11 +303,7 @@ func main() { if nerrors > 0 { os.Exit(2) } - pkg := f.Package - if dir := os.Getenv("CGOPKGPATH"); dir != "" { - pkg = filepath.Join(dir, pkg) - } - p.PackagePath = pkg + p.PackagePath = f.Package p.Record(f) if *godefs { os.Stdout.WriteString(p.godefs(f, input)) diff --git a/libgo/go/cmd/cgo/out.go b/libgo/go/cmd/cgo/out.go index 90a7441..ca0ec0a 100644 --- a/libgo/go/cmd/cgo/out.go +++ b/libgo/go/cmd/cgo/out.go @@ -103,11 +103,19 @@ func (p *Package) writeDefs() { } if *gccgo { + fmt.Fprint(fgo2, gccgoGoProlog) fmt.Fprint(fc, p.cPrologGccgo()) } else { fmt.Fprint(fgo2, goProlog) } + for i, t := range p.CgoChecks { + n := p.unsafeCheckPointerNameIndex(i) + fmt.Fprintf(fgo2, "\nfunc %s(p interface{}, args ...interface{}) %s {\n", n, t) + fmt.Fprintf(fgo2, "\treturn _cgoCheckPointer(p, args...).(%s)\n", t) + fmt.Fprintf(fgo2, "}\n") + } + gccgoSymbolPrefix := p.gccgoSymbolPrefix() cVars := make(map[string]bool) @@ -693,7 +701,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { } fntype := fn.Type forFieldList(fntype.Params, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { t := p.cgoType(atype) if off%t.Align != 0 { pad := t.Align - off%t.Align @@ -711,7 +719,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { npad++ } forFieldList(fntype.Results, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { t := p.cgoType(atype) if off%t.Align != 0 { pad := t.Align - off%t.Align @@ -744,8 +752,12 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName) fmt.Fprintf(fgcch, "struct %s_return {\n", exp.ExpName) forFieldList(fntype.Results, - func(i int, atype ast.Expr) { - fmt.Fprintf(fgcch, "\t%s r%d;\n", p.cgoType(atype).C, i) + func(i int, aname string, atype ast.Expr) { + fmt.Fprintf(fgcch, "\t%s r%d;", p.cgoType(atype).C, i) + if len(aname) > 0 { + fmt.Fprintf(fgcch, " /* %s */", aname) + } + fmt.Fprint(fgcch, "\n") }) fmt.Fprintf(fgcch, "};\n") gccResult = "struct " + exp.ExpName + "_return" @@ -758,7 +770,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { s += " recv" } forFieldList(fntype.Params, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { if i > 0 || fn.Recv != nil { s += ", " } @@ -783,7 +795,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcc, "\ta.recv = recv;\n") } forFieldList(fntype.Params, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { fmt.Fprintf(fgcc, "\ta.p%d = p%d;\n", i, i) }) fmt.Fprintf(fgcc, "\tcrosscall2(_cgoexp%s_%s, &a, %d);\n", cPrefix, exp.ExpName, off) @@ -792,7 +804,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcc, "\treturn a.r0;\n") } else { forFieldList(fntype.Results, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { fmt.Fprintf(fgcc, "\tr.r%d = a.r%d;\n", i, i) }) fmt.Fprintf(fgcc, "\treturn r;\n") @@ -800,17 +812,18 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { } fmt.Fprintf(fgcc, "}\n") - // Build the wrapper function compiled by gc. - goname := exp.Func.Name.Name + // Build the wrapper function compiled by cmd/compile. + goname := "_cgoexpwrap" + cPrefix + "_" if fn.Recv != nil { - goname = "_cgoexpwrap" + cPrefix + "_" + fn.Recv.List[0].Names[0].Name + "_" + goname + goname += fn.Recv.List[0].Names[0].Name + "_" } - fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", goname) + goname += exp.Func.Name.Name + fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", exp.ExpName) fmt.Fprintf(fgo2, "//go:linkname _cgoexp%s_%s _cgoexp%s_%s\n", cPrefix, exp.ExpName, cPrefix, exp.ExpName) fmt.Fprintf(fgo2, "//go:cgo_export_static _cgoexp%s_%s\n", cPrefix, exp.ExpName) fmt.Fprintf(fgo2, "//go:nosplit\n") // no split stack, so no use of m or g fmt.Fprintf(fgo2, "//go:norace\n") // must not have race detector calls inserted - fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a unsafe.Pointer, n int32) {", cPrefix, exp.ExpName) + fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a unsafe.Pointer, n int32) {\n", cPrefix, exp.ExpName) fmt.Fprintf(fgo2, "\tfn := %s\n", goname) // The indirect here is converting from a Go function pointer to a C function pointer. fmt.Fprintf(fgo2, "\t_cgo_runtime_cgocallback(**(**unsafe.Pointer)(unsafe.Pointer(&fn)), a, uintptr(n));\n") @@ -818,44 +831,75 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fm, "int _cgoexp%s_%s;\n", cPrefix, exp.ExpName) - // Calling a function with a receiver from C requires - // a Go wrapper function. + // This code uses printer.Fprint, not conf.Fprint, + // because we don't want //line comments in the middle + // of the function types. + fmt.Fprintf(fgo2, "\n") + fmt.Fprintf(fgo2, "func %s(", goname) + comma := false if fn.Recv != nil { - fmt.Fprintf(fgo2, "func %s(recv ", goname) - conf.Fprint(fgo2, fset, fn.Recv.List[0].Type) - forFieldList(fntype.Params, - func(i int, atype ast.Expr) { - fmt.Fprintf(fgo2, ", p%d ", i) - conf.Fprint(fgo2, fset, atype) - }) - fmt.Fprintf(fgo2, ")") - if gccResult != "void" { - fmt.Fprint(fgo2, " (") - forFieldList(fntype.Results, - func(i int, atype ast.Expr) { - if i > 0 { - fmt.Fprint(fgo2, ", ") - } - conf.Fprint(fgo2, fset, atype) - }) - fmt.Fprint(fgo2, ")") - } - fmt.Fprint(fgo2, " {\n") - fmt.Fprint(fgo2, "\t") - if gccResult != "void" { - fmt.Fprint(fgo2, "return ") - } - fmt.Fprintf(fgo2, "recv.%s(", exp.Func.Name) - forFieldList(fntype.Params, - func(i int, atype ast.Expr) { + fmt.Fprintf(fgo2, "recv ") + printer.Fprint(fgo2, fset, fn.Recv.List[0].Type) + comma = true + } + forFieldList(fntype.Params, + func(i int, aname string, atype ast.Expr) { + if comma { + fmt.Fprintf(fgo2, ", ") + } + fmt.Fprintf(fgo2, "p%d ", i) + printer.Fprint(fgo2, fset, atype) + comma = true + }) + fmt.Fprintf(fgo2, ")") + if gccResult != "void" { + fmt.Fprint(fgo2, " (") + forFieldList(fntype.Results, + func(i int, aname string, atype ast.Expr) { if i > 0 { fmt.Fprint(fgo2, ", ") } - fmt.Fprintf(fgo2, "p%d", i) + fmt.Fprintf(fgo2, "r%d ", i) + printer.Fprint(fgo2, fset, atype) }) - fmt.Fprint(fgo2, ")\n") - fmt.Fprint(fgo2, "}\n") + fmt.Fprint(fgo2, ")") } + fmt.Fprint(fgo2, " {\n") + if gccResult == "void" { + fmt.Fprint(fgo2, "\t") + } else { + // Verify that any results don't contain any + // Go pointers. + addedDefer := false + forFieldList(fntype.Results, + func(i int, aname string, atype ast.Expr) { + if !p.hasPointer(nil, atype, false) { + return + } + if !addedDefer { + fmt.Fprint(fgo2, "\tdefer func() {\n") + addedDefer = true + } + fmt.Fprintf(fgo2, "\t\t_cgoCheckResult(r%d)\n", i) + }) + if addedDefer { + fmt.Fprint(fgo2, "\t}()\n") + } + fmt.Fprint(fgo2, "\treturn ") + } + if fn.Recv != nil { + fmt.Fprintf(fgo2, "recv.") + } + fmt.Fprintf(fgo2, "%s(", exp.Func.Name) + forFieldList(fntype.Params, + func(i int, aname string, atype ast.Expr) { + if i > 0 { + fmt.Fprint(fgo2, ", ") + } + fmt.Fprintf(fgo2, "p%d", i) + }) + fmt.Fprint(fgo2, ")\n") + fmt.Fprint(fgo2, "}\n") } fmt.Fprintf(fgcch, "%s", gccExportHeaderEpilog) @@ -879,13 +923,13 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { cdeclBuf := new(bytes.Buffer) resultCount := 0 forFieldList(fntype.Results, - func(i int, atype ast.Expr) { resultCount++ }) + func(i int, aname string, atype ast.Expr) { resultCount++ }) switch resultCount { case 0: fmt.Fprintf(cdeclBuf, "void") case 1: forFieldList(fntype.Results, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { t := p.cgoType(atype) fmt.Fprintf(cdeclBuf, "%s", t.C) }) @@ -894,9 +938,13 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcch, "\n/* Return type for %s */\n", exp.ExpName) fmt.Fprintf(fgcch, "struct %s_result {\n", exp.ExpName) forFieldList(fntype.Results, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { t := p.cgoType(atype) - fmt.Fprintf(fgcch, "\t%s r%d;\n", t.C, i) + fmt.Fprintf(fgcch, "\t%s r%d;", t.C, i) + if len(aname) > 0 { + fmt.Fprintf(fgcch, " /* %s */", aname) + } + fmt.Fprint(fgcch, "\n") }) fmt.Fprintf(fgcch, "};\n") fmt.Fprintf(cdeclBuf, "struct %s_result", exp.ExpName) @@ -911,7 +959,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { } // Function parameters. forFieldList(fntype.Params, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { if i > 0 || fn.Recv != nil { fmt.Fprintf(cdeclBuf, ", ") } @@ -925,23 +973,15 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcch, "\n%s", exp.Doc) } + fmt.Fprintf(fgcch, "extern %s %s %s;\n", cRet, exp.ExpName, cParams) + // We need to use a name that will be exported by the // Go code; otherwise gccgo will make it static and we // will not be able to link against it from the C // code. goName := "Cgoexp_" + exp.ExpName - fmt.Fprintf(fgcch, `extern %s %s %s __asm__("%s.%s");`, cRet, goName, cParams, gccgoSymbolPrefix, goName) - fmt.Fprint(fgcch, "\n") - - // Use a #define so that the C code that includes - // cgo_export.h will be able to refer to the Go - // function using the expected name. - fmt.Fprintf(fgcch, "#define %s %s\n", exp.ExpName, goName) - - // Use a #undef in _cgo_export.c so that we ignore the - // #define from cgo_export.h, since here we are - // defining the real function. - fmt.Fprintf(fgcc, "#undef %s\n", exp.ExpName) + fmt.Fprintf(fgcc, `extern %s %s %s __asm__("%s.%s");`, cRet, goName, cParams, gccgoSymbolPrefix, goName) + fmt.Fprint(fgcc, "\n") fmt.Fprint(fgcc, "\n") fmt.Fprintf(fgcc, "%s %s %s {\n", cRet, exp.ExpName, cParams) @@ -956,7 +996,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprint(fgcc, "recv") } forFieldList(fntype.Params, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { if i > 0 || fn.Recv != nil { fmt.Fprintf(fgcc, ", ") } @@ -982,7 +1022,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { printer.Fprint(fgo2, fset, fn.Recv.List[0].Type) } forFieldList(fntype.Params, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { if i > 0 || fn.Recv != nil { fmt.Fprintf(fgo2, ", ") } @@ -993,7 +1033,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { if resultCount > 0 { fmt.Fprintf(fgo2, " (") forFieldList(fntype.Results, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { if i > 0 { fmt.Fprint(fgo2, ", ") } @@ -1013,7 +1053,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { } fmt.Fprintf(fgo2, "%s(", exp.Func.Name) forFieldList(fntype.Params, - func(i int, atype ast.Expr) { + func(i int, aname string, atype ast.Expr) { if i > 0 { fmt.Fprint(fgo2, ", ") } @@ -1071,19 +1111,19 @@ func (p *Package) gccgoSymbolPrefix() string { } // Call a function for each entry in an ast.FieldList, passing the -// index into the list and the type. -func forFieldList(fl *ast.FieldList, fn func(int, ast.Expr)) { +// index into the list, the name if any, and the type. +func forFieldList(fl *ast.FieldList, fn func(int, string, ast.Expr)) { if fl == nil { return } i := 0 for _, r := range fl.List { if r.Names == nil { - fn(i, r.Type) + fn(i, "", r.Type) i++ } else { - for range r.Names { - fn(i, r.Type) + for _, n := range r.Names { + fn(i, n.Name, r.Type) i++ } } @@ -1193,9 +1233,11 @@ func (p *Package) cgoType(e ast.Expr) *Type { } const gccProlog = ` -// Usual nonsense: if x and y are not equal, the type will be invalid -// (have a negative array count) and an inscrutable error will come -// out of the compiler and hopefully mention "name". +/* + If x and y are not equal, the type will be invalid + (have a negative array count) and an inscrutable error will come + out of the compiler and hopefully mention "name". +*/ #define __cgo_compile_assert_eq(x, y, name) typedef char name[(x-y)*(x-y)*-2+1]; // Check at compile time that the sizes we use match our expectations. @@ -1239,6 +1281,18 @@ func _cgo_runtime_cmalloc(uintptr) unsafe.Pointer //go:linkname _cgo_runtime_cgocallback runtime.cgocallback func _cgo_runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr) + +//go:linkname _cgoCheckPointer runtime.cgoCheckPointer +func _cgoCheckPointer(interface{}, ...interface{}) interface{} + +//go:linkname _cgoCheckResult runtime.cgoCheckResult +func _cgoCheckResult(interface{}) +` + +const gccgoGoProlog = ` +func _cgoCheckPointer(interface{}, ...interface{}) interface{} + +func _cgoCheckResult(interface{}) ` const goStringDef = ` @@ -1293,7 +1347,8 @@ var builtinDefs = map[string]string{ } func (p *Package) cPrologGccgo() string { - return strings.Replace(cPrologGccgo, "PREFIX", cPrefix, -1) + return strings.Replace(strings.Replace(cPrologGccgo, "PREFIX", cPrefix, -1), + "GCCGOSYMBOLPREF", p.gccgoSymbolPrefix(), -1) } const cPrologGccgo = ` @@ -1348,6 +1403,39 @@ void *_cgoPREFIX_Cfunc__CMalloc(size_t n) { runtime_throw("runtime: C malloc failed"); return p; } + +struct __go_type_descriptor; +typedef struct __go_empty_interface { + const struct __go_type_descriptor *__type_descriptor; + void *__object; +} Eface; + +extern Eface runtimeCgoCheckPointer(Eface, Slice) + __asm__("runtime.cgoCheckPointer") + __attribute__((weak)); + +extern Eface localCgoCheckPointer(Eface, Slice) + __asm__("GCCGOSYMBOLPREF._cgoCheckPointer"); + +Eface localCgoCheckPointer(Eface ptr, Slice args) { + if(runtimeCgoCheckPointer) { + return runtimeCgoCheckPointer(ptr, args); + } + return ptr; +} + +extern void runtimeCgoCheckResult(Eface) + __asm__("runtime.cgoCheckResult") + __attribute__((weak)); + +extern void localCgoCheckResult(Eface) + __asm__("GCCGOSYMBOLPREF._cgoCheckResult"); + +void localCgoCheckResult(Eface val) { + if(runtimeCgoCheckResult) { + runtimeCgoCheckResult(val); + } +} ` func (p *Package) gccExportHeaderProlog() string { @@ -1373,14 +1461,16 @@ typedef GoUintGOINTBITS GoUint; typedef __SIZE_TYPE__ GoUintptr; typedef float GoFloat32; typedef double GoFloat64; -typedef __complex float GoComplex64; -typedef __complex double GoComplex128; +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; -// static assertion to make sure the file is being used on architecture -// at least with matching size of GoInt. +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ typedef char _check_for_GOINTBITS_bit_pointer_matching_GoInt[sizeof(void*)==GOINTBITS/8 ? 1:-1]; -typedef struct { char *p; GoInt n; } GoString; +typedef struct { const char *p; GoInt n; } GoString; typedef void *GoMap; typedef void *GoChan; typedef struct { void *t; void *v; } GoInterface; diff --git a/libgo/go/cmd/go/alldocs.go b/libgo/go/cmd/go/alldocs.go index 1134997..5db4bc6 100644 --- a/libgo/go/cmd/go/alldocs.go +++ b/libgo/go/cmd/go/alldocs.go @@ -84,12 +84,16 @@ and test commands: -n print the commands but do not run them. -p n - the number of builds that can be run in parallel. + the number of programs, such as build commands or + test binaries, that can be run in parallel. The default is the number of CPUs available, except on darwin/arm which defaults to 1. -race enable data race detection. Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. + -msan + enable interoperation with memory sanitizer. + Supported only on linux/amd64. -v print the names of packages as they are compiled. -work @@ -112,13 +116,14 @@ and test commands: a suffix to use in the name of the package installation directory, in order to keep output separate from default builds. If using the -race flag, the install suffix is automatically set to race - or, if set explicitly, has _race appended to it. Using a -buildmode - option that requires non-default compile flags has a similar effect. + or, if set explicitly, has _race appended to it. Likewise for the -msan + flag. Using a -buildmode option that requires non-default compile flags + has a similar effect. -ldflags 'flag list' arguments to pass on each go tool link invocation. -linkshared link against shared libraries previously created with - -buildmode=shared + -buildmode=shared. -pkgdir dir install and load all packages from dir instead of the usual locations. For example, when building with a non-standard configuration, @@ -225,12 +230,17 @@ which is schematically one of these: go doc <pkg> go doc <sym>[.<method>] - go doc [<pkg>].<sym>[.<method>] + go doc [<pkg>.]<sym>[.<method>] + go doc [<pkg>.][<sym>.]<method> -The first item in this list matched by the argument is the one whose -documentation is printed. (See the examples below.) For packages, the order of -scanning is determined lexically, but the GOROOT tree is always scanned before -GOPATH. +The first item in this list matched by the argument is the one whose documentation +is printed. (See the examples below.) However, if the argument starts with a capital +letter it is assumed to identify a symbol or method in the current directory. + +For packages, the order of scanning is determined lexically in breadth-first order. +That is, the package presented is the one that matches the search and is nearest +the root and lexically first at its level of the hierarchy. The GOROOT tree is +always scanned in its entirety before GOPATH. If there is no package specified or matched, the package in the current directory is selected, so "go doc Foo" shows the documentation for symbol Foo in @@ -278,6 +288,14 @@ Examples: go doc text/template new # Two arguments Show documentation for text/template's New function. + At least in the current tree, these invocations all print the + documentation for json.Decoder's Decode method: + + go doc json.Decoder.Decode + go doc json.decoder.decode + go doc json.decode + cd go/src/encoding/json; go doc decode + Flags: -c Respect case when matching symbols. @@ -344,7 +362,7 @@ Generate Go files by processing source Usage: - go generate [-run regexp] [file.go... | packages] + go generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages] Generate runs commands described by directives within existing files. Those commands can run any process but the intent is to @@ -436,12 +454,14 @@ Go generate accepts one specific flag: any trailing spaces and final newline) matches the expression. -It also accepts the standard build flags -v, -n, and -x. +It also accepts the standard build flags including -v, -n, and -x. The -v flag prints the names of packages and files as they are processed. The -n flag prints commands that would be executed. The -x flag prints commands as they are executed. +For more about build flags, see 'go help build'. + For more about specifying packages, see 'go help packages'. @@ -477,16 +497,22 @@ missing packages but does not use it to look for updates to existing packages. Get also accepts build flags to control the installation. See 'go help build'. +When checking out a new package, get creates the target directory +GOPATH/src/<import-path>. If the GOPATH contains multiple entries, +get uses the first one. See 'go help gopath'. + When checking out or updating a package, get looks for a branch or tag that matches the locally installed version of Go. The most important rule is that if the local installation is running version "go1", get searches for a branch or tag named "go1". If no such version exists it retrieves the most recent version of the package. -If the vendoring experiment is enabled (see 'go help gopath'), -then when go get checks out or updates a Git repository, +Unless vendoring support is disabled (see 'go help gopath'), +when go get checks out or updates a Git repository, it also updates any git submodules referenced by the repository. +Get never checks out or updates code stored in vendor directories. + For more about specifying packages, see 'go help packages'. For more about how 'go get' finds source code to @@ -577,6 +603,14 @@ syntax of package template. The default output is equivalent to -f XTestImports []string // imports from XTestGoFiles } +The error information, if any, is + + type PackageError struct { + ImportStack []string // shortest path from package named on command line to this one + Pos string // position of error (if present, file:line:col) + Err string // the error itself + } + The template function "join" calls strings.Join. The template function "context" returns the build context, defined as: @@ -643,7 +677,7 @@ Test packages Usage: - go test [-c] [-i] [build and test flags] [packages] [flags for test binary] + go test [build/test flags] [packages] [build/test flags & test binary flags] 'Go test' automates testing the packages named by the import paths. It prints a summary of the test results in the format: @@ -673,10 +707,16 @@ non-test installation. In addition to the build flags, the flags handled by 'go test' itself are: + -args + Pass the remainder of the command line (everything after -args) + to the test binary, uninterpreted and unchanged. + Because this flag consumes the remainder of the command line, + the package list (if present) must appear before this flag. + -c - Compile the test binary to pkg.test but do not run it - (where pkg is the last element of the package's import path). - The file name can be changed with the -o flag. + Compile the test binary to pkg.test but do not run it + (where pkg is the last element of the package's import path). + The file name can be changed with the -o flag. -exec xprog Run the test binary using xprog. The behavior is the same as @@ -687,17 +727,12 @@ In addition to the build flags, the flags handled by 'go test' itself are: Do not run the test. -o file - Compile the test binary to the named file. - The test still runs (unless -c or -i is specified). + Compile the test binary to the named file. + The test still runs (unless -c or -i is specified). The test binary also accepts flags that control execution of the test; these flags are also accessible by 'go test'. See 'go help testflag' for details. -If the test binary needs any other flags, they should be presented after the -package names. The go tool treats as a flag the first argument that begins with -a minus sign that it does not recognize itself; that argument and all subsequent -arguments are passed as arguments to the test binary. - For more about build flags, see 'go help build'. For more about specifying packages, see 'go help packages'. @@ -804,6 +839,11 @@ are: Build the listed main packages and everything they import into executables. Packages not named main are ignored. + -buildmode=pie + Build the listed main packages and everything they import into + position independent executables (PIE). Packages not named + main are ignored. + File types @@ -875,7 +915,7 @@ DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped so that you can add DIR/bin to your PATH to get at the installed commands. If the GOBIN environment variable is set, commands are installed to the directory it names instead -of DIR/bin. +of DIR/bin. GOBIN must be an absolute path. Here's an example directory layout: @@ -933,13 +973,10 @@ See https://golang.org/s/go14internal for details. Vendor Directories -Go 1.5 includes experimental support for using local copies -of external dependencies to satisfy imports of those dependencies, -often referred to as vendoring. Setting the environment variable -GO15VENDOREXPERIMENT=1 enables that experimental support. +Go 1.6 includes support for using local copies of external dependencies +to satisfy imports of those dependencies, often referred to as vendoring. -When the vendor experiment is enabled, -code below a directory named "vendor" is importable only +Code below a directory named "vendor" is importable only by code in the directory tree rooted at the parent of "vendor", and only using an import path that omits the prefix up to and including the vendor element. @@ -977,12 +1014,18 @@ top-level "crash/bang". Code in vendor directories is not subject to import path checking (see 'go help importpath'). -When the vendor experiment is enabled, 'go get' checks out -submodules when checking out or updating a git repository -(see 'go help get'). +When 'go get' checks out or updates a git repository, it now also +updates submodules. + +Vendor directories do not affect the placement of new repositories +being checked out for the first time by 'go get': those are always +placed in the main GOPATH, never in a vendor subtree. -The vendoring semantics are an experiment, and they may change -in future releases. Once settled, they will be on by default. +In Go 1.5, as an experiment, setting the environment variable +GO15VENDOREXPERIMENT=1 enabled these features. +As of Go 1.6 they are on by default. To turn them off, set +GO15VENDOREXPERIMENT=0. In Go 1.7, the environment +variable will stop having any effect. See https://golang.org/s/go15vendor for details. @@ -1051,7 +1094,7 @@ Special-purpose environment variables: File names in stack traces are rewritten from GOROOT to GOROOT_FINAL. GO15VENDOREXPERIMENT - Set to 1 to enable the Go 1.5 vendoring experiment. + Set to 0 to disable vendoring semantics. GO_EXTLINK_ENABLED Whether the linker should use external linking mode when using -linkmode=auto with code that uses cgo. @@ -1227,10 +1270,10 @@ unless it is being referred to by that import path. In this way, import comments let package authors make sure the custom import path is used and not a direct path to the underlying code hosting site. -If the vendoring experiment is enabled (see 'go help gopath'), -then import path checking is disabled for code found within vendor trees. -This makes it possible to copy code into alternate locations in vendor trees -without needing to update import comments. +If vendoring is enabled (see 'go help gopath'), then import path checking is +disabled for code found within vendor trees. This makes it possible to copy +code into alternate locations in vendor trees without needing to update import +comments. See https://golang.org/s/go14customimport for details. @@ -1286,6 +1329,14 @@ internally at Google all begin with 'google', and paths denoting remote repositories begin with the path to the code, such as 'github.com/user/repo'. +Packages in a program need not have unique package names, +but there are two reserved package names with special meaning. +The name main indicates a command, not a library. +Commands are built into binaries and cannot be imported. +The name documentation indicates documentation for +a non-Go program in the directory. Files in package documentation +are ignored by the go command. + As a special case, if the package list is a list of .go files from a single directory, the command is applied to a single synthesized package made up of exactly those files, ignoring any build constraints @@ -1391,6 +1442,10 @@ control the execution of any test: Allow parallel execution of test functions that call t.Parallel. The value of this flag is the maximum number of tests to run simultaneously; by default, it is set to the value of GOMAXPROCS. + Note that -parallel only applies within a single test binary. + The 'go test' command may run tests for different packages + in parallel as well, according to the setting of the -p flag + (see 'go help build'). -run regexp Run only those tests and examples matching the regular @@ -1414,25 +1469,63 @@ control the execution of any test: Verbose output: log all tests as they are run. Also print all text from Log and Logf calls even if the test succeeds. -The test binary, called pkg.test where pkg is the name of the -directory containing the package sources, can be invoked directly -after building it with 'go test -c'. When invoking the test binary -directly, each of the standard flag names must be prefixed with 'test.', -as in -test.run=TestMyFunc or -test.v. +Each of these flags is also recognized with an optional 'test.' prefix, +as in -test.v. When invoking the generated test binary (the result of +'go test -c') directly, however, the prefix is mandatory. + +The 'go test' command rewrites or removes recognized flags, +as appropriate, both before and after the optional package list, +before invoking the test binary. -When running 'go test', flags not listed above are passed through -unaltered. For instance, the command +For instance, the command - go test -x -v -cpuprofile=prof.out -dir=testdata -update + go test -v -myflag testdata -cpuprofile=prof.out -x will compile the test binary and then run it as - pkg.test -test.v -test.cpuprofile=prof.out -dir=testdata -update + pkg.test -test.v -myflag testdata -test.cpuprofile=prof.out + +(The -x flag is removed because it applies only to the go command's +execution, not to the test itself.) The test flags that generate profiles (other than for coverage) also leave the test binary in pkg.test for use when analyzing the profiles. -Flags not recognized by 'go test' must be placed after any specified packages. +When 'go test' runs a test binary, it does so from within the +corresponding package's source code directory. Depending on the test, +it may be necessary to do the same when invoking a generated test +binary directly. + +The command-line package list, if present, must appear before any +flag not known to the go test command. Continuing the example above, +the package list would have to appear before -myflag, but could appear +on either side of -v. + +To keep an argument for a test binary from being interpreted as a +known flag or a package name, use -args (see 'go help test') which +passes the remainder of the command line through to the test binary +uninterpreted and unaltered. + +For instance, the command + + go test -v -args -x -v + +will compile the test binary and then run it as + + pkg.test -test.v -x -v + +Similarly, + + go test -args math + +will compile the test binary and then run it as + + pkg.test math + +In the first example, the -x and the second -v are passed through to the +test binary unchanged and with no effect on the go command itself. +In the second example, the argument math is passed through to the test +binary, instead of being interpreted as the package list. Description of testing functions diff --git a/libgo/go/cmd/go/build.go b/libgo/go/cmd/go/build.go index 865871c..3dd9df4 100644 --- a/libgo/go/cmd/go/build.go +++ b/libgo/go/cmd/go/build.go @@ -63,12 +63,16 @@ and test commands: -n print the commands but do not run them. -p n - the number of builds that can be run in parallel. + the number of programs, such as build commands or + test binaries, that can be run in parallel. The default is the number of CPUs available, except on darwin/arm which defaults to 1. -race enable data race detection. Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. + -msan + enable interoperation with memory sanitizer. + Supported only on linux/amd64. -v print the names of packages as they are compiled. -work @@ -91,13 +95,14 @@ and test commands: a suffix to use in the name of the package installation directory, in order to keep output separate from default builds. If using the -race flag, the install suffix is automatically set to race - or, if set explicitly, has _race appended to it. Using a -buildmode - option that requires non-default compile flags has a similar effect. + or, if set explicitly, has _race appended to it. Likewise for the -msan + flag. Using a -buildmode option that requires non-default compile flags + has a similar effect. -ldflags 'flag list' arguments to pass on each go tool link invocation. -linkshared link against shared libraries previously created with - -buildmode=shared + -buildmode=shared. -pkgdir dir install and load all packages from dir instead of the usual locations. For example, when building with a non-standard configuration, @@ -166,6 +171,7 @@ var buildGcflags []string // -gcflags flag var buildLdflags []string // -ldflags flag var buildGccgoflags []string // -gccgoflags flag var buildRace bool // -race flag +var buildMSan bool // -msan flag var buildToolExec []string // -toolexec flag var buildBuildmode string // -buildmode flag var buildLinkshared bool // -linkshared flag @@ -227,6 +233,7 @@ func addBuildFlags(cmd *Command) { cmd.Flag.BoolVar(&buildLinkshared, "linkshared", false, "") cmd.Flag.StringVar(&buildPkgdir, "pkgdir", "", "") cmd.Flag.BoolVar(&buildRace, "race", false, "") + cmd.Flag.BoolVar(&buildMSan, "msan", false, "") cmd.Flag.Var((*stringsFlag)(&buildContext.BuildTags), "tags", "") cmd.Flag.Var((*stringsFlag)(&buildToolExec), "toolexec", "") cmd.Flag.BoolVar(&buildWork, "work", false, "") @@ -352,29 +359,46 @@ func buildModeInit() { codegenArg = "-fPIC" } else { switch platform { - case "linux/amd64": + case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", + "android/amd64", "android/arm", "android/arm64", "android/386": codegenArg = "-shared" - case "linux/arm": - buildAsmflags = append(buildAsmflags, "-shared") - case "darwin/amd64": - case "android/arm": + case "darwin/amd64", "darwin/386": default: fatalf("-buildmode=c-shared not supported on %s\n", platform) } } ldBuildmode = "c-shared" case "default": - ldBuildmode = "exe" + switch platform { + case "android/arm", "android/arm64", "android/amd64", "android/386": + codegenArg = "-shared" + ldBuildmode = "pie" + default: + ldBuildmode = "exe" + } case "exe": pkgsFilter = pkgsMain ldBuildmode = "exe" + case "pie": + if gccgo { + fatalf("-buildmode=pie not supported by gccgo") + } else { + switch platform { + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le", + "android/amd64", "android/arm", "android/arm64", "android/386": + codegenArg = "-shared" + default: + fatalf("-buildmode=pie not supported on %s\n", platform) + } + } + ldBuildmode = "pie" case "shared": pkgsFilter = pkgsNotMain if gccgo { codegenArg = "-fPIC" } else { switch platform { - case "linux/amd64": + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le": default: fatalf("-buildmode=shared not supported on %s\n", platform) } @@ -391,9 +415,11 @@ func buildModeInit() { if gccgo { codegenArg = "-fPIC" } else { - if platform != "linux/amd64" { - fmt.Fprintf(os.Stderr, "go %s: -linkshared is only supported on linux/amd64\n", flag.Args()[0]) - os.Exit(2) + switch platform { + case "linux/386", "linux/amd64", "linux/arm", "linux/arm64", "linux/ppc64le": + buildAsmflags = append(buildAsmflags, "-D=GOBUILDMODE_shared=1") + default: + fatalf("-linkshared not supported on %s\n", platform) } codegenArg = "-dynlink" // TODO(mwhudson): remove -w when that gets fixed in linker. @@ -415,7 +441,7 @@ func buildModeInit() { } func runBuild(cmd *Command, args []string) { - raceInit() + instrumentInit() buildModeInit() var b builder b.init() @@ -463,7 +489,12 @@ func runBuild(cmd *Command, args []string) { var a *action if buildBuildmode == "shared" { - a = b.libaction(libname(args), pkgsFilter(packages(args)), modeBuild, depMode) + pkgs := pkgsFilter(packages(args)) + if libName, err := libname(args, pkgs); err != nil { + fatalf("%s", err.Error()) + } else { + a = b.libaction(libName, pkgs, modeBuild, depMode) + } } else { a = &action{} for _, p := range pkgsFilter(packages(args)) { @@ -487,32 +518,73 @@ See also: go build, go get, go clean. `, } +// isMetaPackage checks if name is a reserved package name that expands to multiple packages +func isMetaPackage(name string) bool { + return name == "std" || name == "cmd" || name == "all" +} + // libname returns the filename to use for the shared library when using // -buildmode=shared. The rules we use are: -// 1) Drop any trailing "/..."s if present -// 2) Change / to - -// 3) Join arguments with , -// So std -> libstd.so -// a b/... -> liba,b.so -// gopkg.in/tomb.v2 -> libgopkg.in-tomb.v2.so -func libname(args []string) string { +// Use arguments for special 'meta' packages: +// std --> libstd.so +// std cmd --> libstd,cmd.so +// A single non-meta argument with trailing "/..." is special cased: +// foo/... --> libfoo.so +// (A relative path like "./..." expands the "." first) +// Use import paths for other cases, changing '/' to '-': +// somelib --> libsubdir-somelib.so +// ./ or ../ --> libsubdir-somelib.so +// gopkg.in/tomb.v2 -> libgopkg.in-tomb.v2.so +// a/... b/... ---> liba/c,b/d.so - all matching import paths +// Name parts are joined with ','. +func libname(args []string, pkgs []*Package) (string, error) { var libname string - for _, arg := range args { - arg = strings.TrimSuffix(arg, "/...") - arg = strings.Replace(arg, "/", "-", -1) + appendName := func(arg string) { if libname == "" { libname = arg } else { libname += "," + arg } } + var haveNonMeta bool + for _, arg := range args { + if isMetaPackage(arg) { + appendName(arg) + } else { + haveNonMeta = true + } + } + if len(libname) == 0 { // non-meta packages only. use import paths + if len(args) == 1 && strings.HasSuffix(args[0], "/...") { + // Special case of "foo/..." as mentioned above. + arg := strings.TrimSuffix(args[0], "/...") + if build.IsLocalImport(arg) { + cwd, _ := os.Getwd() + bp, _ := buildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly) + if bp.ImportPath != "" && bp.ImportPath != "." { + arg = bp.ImportPath + } + } + appendName(strings.Replace(arg, "/", "-", -1)) + } else { + for _, pkg := range pkgs { + appendName(strings.Replace(pkg.ImportPath, "/", "-", -1)) + } + } + } else if haveNonMeta { // have both meta package and a non-meta one + return "", errors.New("mixing of meta and non-meta packages is not allowed") + } // TODO(mwhudson): Needs to change for platforms that use different naming // conventions... - return "lib" + libname + ".so" + return "lib" + libname + ".so", nil } func runInstall(cmd *Command, args []string) { - raceInit() + if gobin != "" && !filepath.IsAbs(gobin) { + fatalf("cannot install, GOBIN must be an absolute path") + } + + instrumentInit() buildModeInit() pkgs := pkgsFilter(packagesForBuild(args)) @@ -537,7 +609,11 @@ func runInstall(cmd *Command, args []string) { b.init() var a *action if buildBuildmode == "shared" { - a = b.libaction(libname(args), pkgs, modeInstall, modeInstall) + if libName, err := libname(args, pkgs); err != nil { + fatalf("%s", err.Error()) + } else { + a = b.libaction(libName, pkgs, modeInstall, modeInstall) + } } else { a = &action{} var tools []*action @@ -754,7 +830,9 @@ func goFilesPackage(gofiles []string) *Package { pkg := new(Package) pkg.local = true pkg.cmdline = true + stk.push("main") pkg.load(&stk, bp, err) + stk.pop() pkg.localPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" pkg.target = "" @@ -812,15 +890,17 @@ func readpkglist(shlibpath string) (pkgs []*Package) { // action returns the action for applying the given operation (mode) to the package. // depMode is the action to use when building dependencies. -// action never looks for p in a shared library. +// action never looks for p in a shared library, but may find p's dependencies in a +// shared library if buildLinkshared is true. func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action { - return b.action1(mode, depMode, p, false) + return b.action1(mode, depMode, p, false, "") } // action1 returns the action for applying the given operation (mode) to the package. // depMode is the action to use when building dependencies. // action1 will look for p in a shared library if lookshared is true. -func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, lookshared bool) *action { +// forShlib is the shared library that p will become part of, if any. +func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, lookshared bool, forShlib string) *action { shlib := "" if lookshared { shlib = p.Shlib @@ -852,13 +932,23 @@ func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, looksha b.actionCache[key] = a for _, p1 := range p.imports { - ls := buildLinkshared - // If p1 is part of the same shared library as p, we need the action - // that builds p here, not the shared libary or we get action loops. - if p1.Shlib == p.Shlib { - ls = false + if forShlib != "" { + // p is part of a shared library. + if p1.Shlib != "" && p1.Shlib != forShlib { + // p1 is explicitly part of a different shared library. + // Put the action for that shared library into a.deps. + a.deps = append(a.deps, b.action1(depMode, depMode, p1, true, p1.Shlib)) + } else { + // p1 is (implicitly or not) part of this shared library. + // Put the action for p1 into a.deps. + a.deps = append(a.deps, b.action1(depMode, depMode, p1, false, forShlib)) + } + } else { + // p is not part of a shared library. + // If p1 is in a shared library, put the action for that into + // a.deps, otherwise put the action for p1 into a.deps. + a.deps = append(a.deps, b.action1(depMode, depMode, p1, buildLinkshared, p1.Shlib)) } - a.deps = append(a.deps, b.action1(depMode, depMode, p1, ls)) } // If we are not doing a cross-build, then record the binary we'll @@ -866,7 +956,7 @@ func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, looksha // using cgo, to make sure we do not overwrite the binary while // a package is using it. If this is a cross-build, then the cgo we // are writing is not the cgo we need to use. - if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace && reqStdPkgSrc { + if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace && !buildMSan && reqStdPkgSrc { if (len(p.CgoFiles) > 0 || p.Standard && p.ImportPath == "runtime/cgo") && !buildLinkshared && buildBuildmode != "shared" { var stk importStack p1 := loadPackage("cmd/cgo", &stk) @@ -914,18 +1004,27 @@ func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, looksha switch mode { case modeInstall: a.f = (*builder).install - a.deps = []*action{b.action1(modeBuild, depMode, p, lookshared)} + a.deps = []*action{b.action1(modeBuild, depMode, p, lookshared, forShlib)} a.target = a.p.target // Install header for cgo in c-archive and c-shared modes. if p.usesCgo() && (buildBuildmode == "c-archive" || buildBuildmode == "c-shared") { + hdrTarget := a.target[:len(a.target)-len(filepath.Ext(a.target))] + ".h" + if buildContext.Compiler == "gccgo" { + // For the header file, remove the "lib" + // added by go/build, so we generate pkg.h + // rather than libpkg.h. + dir, file := filepath.Split(hdrTarget) + file = strings.TrimPrefix(file, "lib") + hdrTarget = filepath.Join(dir, file) + } ah := &action{ p: a.p, deps: []*action{a.deps[0]}, f: (*builder).installHeader, pkgdir: a.pkgdir, objdir: a.objdir, - target: a.target[:len(a.target)-len(filepath.Ext(a.target))] + ".h", + target: hdrTarget, } a.deps = append(a.deps, ah) } @@ -961,7 +1060,11 @@ func (b *builder) action1(mode buildMode, depMode buildMode, p *Package, looksha func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode buildMode) *action { a := &action{} - if mode == modeBuild { + switch mode { + default: + fatalf("unrecognized mode %v", mode) + + case modeBuild: a.f = (*builder).linkShared a.target = filepath.Join(b.work, libname) for _, p := range pkgs { @@ -970,14 +1073,15 @@ func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode build } a.deps = append(a.deps, b.action(depMode, depMode, p)) } - } else if mode == modeInstall { + + case modeInstall: // Currently build mode shared forces external linking mode, and - // external linking mode forces an import of runtime/cgo. So if it - // was not passed on the command line and it is not present in - // another shared library, add it here. - seencgo := false + // external linking mode forces an import of runtime/cgo (and + // math on arm). So if it was not passed on the command line and + // it is not present in another shared library, add it here. _, gccgo := buildToolchain.(gccgoToolchain) if !gccgo { + seencgo := false for _, p := range pkgs { seencgo = seencgo || (p.Standard && p.ImportPath == "runtime/cgo") } @@ -997,6 +1101,28 @@ func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode build pkgs = append(pkgs, p) } } + if goarch == "arm" { + seenmath := false + for _, p := range pkgs { + seenmath = seenmath || (p.Standard && p.ImportPath == "math") + } + if !seenmath { + var stk importStack + p := loadPackage("math", &stk) + if p.Error != nil { + fatalf("load math: %v", p.Error) + } + computeStale(p) + // If math is in another shared library, then that's + // also the shared library that contains runtime, so + // something will depend on it and so math's staleness + // will be checked when processing that library. + if p.Shlib == "" || p.Shlib == libname { + pkgs = append([]*Package{}, pkgs...) + pkgs = append(pkgs, p) + } + } + } } // Figure out where the library will go. @@ -1029,7 +1155,7 @@ func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode build if err != nil || lstat.ModTime().After(built) { stale = true } - a.deps = append(a.deps, b.action(depMode, depMode, p)) + a.deps = append(a.deps, b.action1(depMode, depMode, p, false, a.target)) } if stale { @@ -1047,8 +1173,6 @@ func (b *builder) libaction(libname string, pkgs []*Package, mode, depMode build shlibnameaction.deps = append(shlibnameaction.deps, buildAction) } } - } else { - fatalf("unregonized mode %v", mode) } return a } @@ -1239,17 +1363,11 @@ func (b *builder) build(a *action) (err error) { // different sections of the bootstrap script have to // be merged, the banners give patch something // to use to find its context. - fmt.Printf("\n#\n# %s\n#\n\n", a.p.ImportPath) + b.print("\n#\n# " + a.p.ImportPath + "\n#\n\n") } if buildV { - fmt.Fprintf(os.Stderr, "%s\n", a.p.ImportPath) - } - - if a.p.Standard && a.p.ImportPath == "runtime" && buildContext.Compiler == "gc" && - (!hasString(a.p.GoFiles, "zgoos_"+buildContext.GOOS+".go") || - !hasString(a.p.GoFiles, "zgoarch_"+buildContext.GOARCH+".go")) { - return fmt.Errorf("%s/%s must be bootstrapped using make%v", buildContext.GOOS, buildContext.GOARCH, defaultSuffix()) + b.print(a.p.ImportPath + "\n") } // Make build directory. @@ -1393,17 +1511,17 @@ func (b *builder) build(a *action) (err error) { switch { case strings.HasSuffix(name, _goos_goarch): targ := file[:len(name)-len(_goos_goarch)] + "_GOOS_GOARCH." + ext - if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644, true); err != nil { + if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0666, true); err != nil { return err } case strings.HasSuffix(name, _goarch): targ := file[:len(name)-len(_goarch)] + "_GOARCH." + ext - if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644, true); err != nil { + if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0666, true); err != nil { return err } case strings.HasSuffix(name, _goos): targ := file[:len(name)-len(_goos)] + "_GOOS." + ext - if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0644, true); err != nil { + if err := b.copyFile(a, obj+targ, filepath.Join(a.p.Dir, file), 0666, true); err != nil { return err } } @@ -1492,7 +1610,7 @@ func (b *builder) getPkgConfigFlags(p *Package) (cflags, ldflags []string, err e func (b *builder) installShlibname(a *action) error { a1 := a.deps[0] - err := ioutil.WriteFile(a.target, []byte(filepath.Base(a1.target)+"\n"), 0644) + err := ioutil.WriteFile(a.target, []byte(filepath.Base(a1.target)+"\n"), 0666) if err != nil { return err } @@ -1516,12 +1634,12 @@ func (b *builder) install(a *action) (err error) { } }() a1 := a.deps[0] - perm := os.FileMode(0644) + perm := os.FileMode(0666) if a1.link { switch buildBuildmode { case "c-archive", "c-shared": default: - perm = 0755 + perm = 0777 } } @@ -1595,7 +1713,25 @@ func (b *builder) moveOrCopyFile(a *action, dst, src string, perm os.FileMode, f // If we can update the mode and rename to the dst, do it. // Otherwise fall back to standard copy. - if err := os.Chmod(src, perm); err == nil { + + // The perm argument is meant to be adjusted according to umask, + // but we don't know what the umask is. + // Create a dummy file to find out. + // This avoids build tags and works even on systems like Plan 9 + // where the file mask computation incorporates other information. + mode := perm + f, err := os.OpenFile(filepath.Clean(dst)+"-go-tmp-umask", os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err == nil { + fi, err := f.Stat() + if err == nil { + mode = fi.Mode() & 0777 + } + name := f.Name() + f.Close() + os.Remove(name) + } + + if err := os.Chmod(src, mode); err == nil { if err := os.Rename(src, dst); err == nil { if buildX { b.showcmd("", "mv %s %s", src, dst) @@ -1629,7 +1765,7 @@ func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode, force b if fi.IsDir() { return fmt.Errorf("build output %q already exists and is a directory", dst) } - if !force && !isObject(dst) { + if !force && fi.Mode().IsRegular() && !isObject(dst) { return fmt.Errorf("build output %q already exists and is not an object file", dst) } } @@ -1641,7 +1777,7 @@ func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode, force b } } - os.Remove(dst) + mayberemovefile(dst) df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) if err != nil && toolIsWindows { // Windows does not allow deletion of a binary file @@ -1660,7 +1796,7 @@ func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode, force b _, err = io.Copy(df, sf) df.Close() if err != nil { - os.Remove(dst) + mayberemovefile(dst) return fmt.Errorf("copying %s to %s: %v", src, dst, err) } return nil @@ -1682,7 +1818,7 @@ func (b *builder) installHeader(a *action) error { } } - return b.moveOrCopyFile(a, a.target, src, 0644, true) + return b.moveOrCopyFile(a, a.target, src, 0666, true) } // cover runs, in effect, @@ -1707,6 +1843,7 @@ var objectMagic = [][]byte{ {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00}, // PE (Windows) as generated by 6l/8l and gcc {0x00, 0x00, 0x01, 0xEB}, // Plan 9 i386 {0x00, 0x00, 0x8a, 0x97}, // Plan 9 amd64 + {0x00, 0x00, 0x06, 0x47}, // Plan 9 arm } func isObject(s string) bool { @@ -1725,6 +1862,16 @@ func isObject(s string) bool { return false } +// mayberemovefile removes a file only if it is a regular file +// When running as a user with sufficient privileges, we may delete +// even device files, for example, which is not intended. +func mayberemovefile(s string) { + if fi, err := os.Lstat(s); err == nil && !fi.Mode().IsRegular() { + return + } + os.Remove(s) +} + // fmtcmd formats a command in the manner of fmt.Sprintf but also: // // If dir is non-empty and the script is not in dir right now, @@ -2103,7 +2250,7 @@ func (gcToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool, if p.Name == "main" { gcargs[1] = "main" } - if p.Standard && p.ImportPath == "runtime" { + if p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) { // runtime compiles with a special gc flag to emit // additional reflect type data. gcargs = append(gcargs, "-+") @@ -2208,33 +2355,26 @@ func (gcToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []s for _, f := range ofiles { absOfiles = append(absOfiles, mkAbs(objDir, f)) } - cmd := "c" absAfile := mkAbs(objDir, afile) - appending := false - if _, err := os.Stat(absAfile); err == nil { - appending = true - cmd = "r" - } - cmdline := stringList("pack", cmd, absAfile, absOfiles) + // The archive file should have been created by the compiler. + // Since it used to not work that way, verify. + if _, err := os.Stat(absAfile); err != nil { + fatalf("os.Stat of archive file failed: %v", err) + } - if appending { - if buildN || buildX { - b.showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) - } - if buildN { - return nil - } - if err := packInternal(b, absAfile, absOfiles); err != nil { - b.showOutput(p.Dir, p.ImportPath, err.Error()+"\n") - return errPrintedOutput - } + if buildN || buildX { + cmdline := stringList("pack", "r", absAfile, absOfiles) + b.showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) + } + if buildN { return nil } - - // Need actual pack. - cmdline[0] = tool("pack") - return b.run(p.Dir, p.ImportPath, nil, buildToolExec, cmdline) + if err := packInternal(b, absAfile, absOfiles); err != nil { + b.showOutput(p.Dir, p.ImportPath, err.Error()+"\n") + return errPrintedOutput + } + return nil } func packInternal(b *builder, afile string, ofiles []string) error { @@ -2445,11 +2585,11 @@ func (tools gccgoToolchain) asm(b *builder, p *Package, obj, ofile, sfile string sfile = mkAbs(p.Dir, sfile) defs := []string{"-D", "GOOS_" + goos, "-D", "GOARCH_" + goarch} if pkgpath := gccgoCleanPkgpath(p); pkgpath != "" { - defs = append(defs, `-D`, `GOPKGPATH="`+pkgpath+`"`) + defs = append(defs, `-D`, `GOPKGPATH=`+pkgpath) } defs = tools.maybePIC(defs) defs = append(defs, b.gccArchArgs()...) - return b.run(p.Dir, p.ImportPath, nil, tools.compiler(), "-c", "-I", obj, "-o", ofile, defs, sfile) + return b.run(p.Dir, p.ImportPath, nil, tools.compiler(), "-xassembler-with-cpp", "-I", obj, "-c", "-o", ofile, defs, sfile) } func (gccgoToolchain) pkgpath(basedir string, p *Package) string { @@ -2464,7 +2604,7 @@ func (gccgoToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles for _, f := range ofiles { absOfiles = append(absOfiles, mkAbs(objDir, f)) } - return b.run(p.Dir, p.ImportPath, nil, "ar", "cru", mkAbs(objDir, afile), absOfiles) + return b.run(p.Dir, p.ImportPath, nil, "ar", "rc", mkAbs(objDir, afile), absOfiles) } func (tools gccgoToolchain) ld(b *builder, root *action, out string, allactions []*action, mainpkg string, ofiles []string) error { @@ -2599,6 +2739,10 @@ func (tools gccgoToolchain) ld(b *builder, root *action, out string, allactions // libffi. ldflags = append(ldflags, "-Wl,-r", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive") + if b.gccSupportsNoPie() { + ldflags = append(ldflags, "-no-pie") + } + // We are creating an object file, so we don't want a build ID. ldflags = b.disableBuildID(ldflags) @@ -2606,7 +2750,7 @@ func (tools gccgoToolchain) ld(b *builder, root *action, out string, allactions out = out + ".o" case "c-shared": - ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc") + ldflags = append(ldflags, "-shared", "-nostdlib", "-Wl,--whole-archive", "-lgolibbegin", "-Wl,--no-whole-archive", "-lgo", "-lgcc_s", "-lgcc", "-lc", "-lgcc") default: fatalf("-buildmode=%s not supported for gccgo", ldBuildmode) @@ -2704,43 +2848,6 @@ func gccgoCleanPkgpath(p *Package) string { return strings.Map(clean, gccgoPkgpath(p)) } -// libgcc returns the filename for libgcc, as determined by invoking gcc with -// the -print-libgcc-file-name option. -func (b *builder) libgcc(p *Package) (string, error) { - var buf bytes.Buffer - - gccCmd := b.gccCmd(p.Dir) - - prev := b.print - if buildN { - // In -n mode we temporarily swap out the builder's - // print function to capture the command-line. This - // let's us assign it to $LIBGCC and produce a valid - // buildscript for cgo packages. - b.print = func(a ...interface{}) (int, error) { - return fmt.Fprint(&buf, a...) - } - } - f, err := b.runOut(p.Dir, p.ImportPath, nil, gccCmd, "-print-libgcc-file-name") - if err != nil { - return "", fmt.Errorf("gcc -print-libgcc-file-name: %v (%s)", err, f) - } - if buildN { - s := fmt.Sprintf("LIBGCC=$(%s)\n", buf.Next(buf.Len()-1)) - b.print = prev - b.print(s) - return "$LIBGCC", nil - } - - // The compiler might not be able to find libgcc, and in that case, - // it will simply return "libgcc.a", which is of no use to us. - if !filepath.IsAbs(string(f)) { - return "", nil - } - - return strings.Trim(string(f), "\r\n"), nil -} - // gcc runs the gcc C compiler to create an object from a single C file. func (b *builder) gcc(p *Package, out string, flags []string, cfile string) error { return b.ccompile(p, out, flags, cfile, b.gccCmd(p.Dir)) @@ -2827,6 +2934,36 @@ func (b *builder) ccompilerCmd(envvar, defcmd, objdir string) []string { return a } +// On systems with PIE (position independent executables) enabled by default, +// -no-pie must be passed when doing a partial link with -Wl,-r. But -no-pie is +// not supported by all compilers. +func (b *builder) gccSupportsNoPie() bool { + if goos != "linux" { + // On some BSD platforms, error messages from the + // compiler make it to the console despite cmd.Std* + // all being nil. As -no-pie is only required on linux + // systems so far, we only test there. + return false + } + src := filepath.Join(b.work, "trivial.c") + if err := ioutil.WriteFile(src, []byte{}, 0666); err != nil { + return false + } + cmdArgs := b.gccCmd(b.work) + cmdArgs = append(cmdArgs, "-no-pie", "-c", "trivial.c") + if buildN || buildX { + b.showcmd(b.work, "%s", joinUnambiguously(cmdArgs)) + if buildN { + return false + } + } + cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) + cmd.Dir = b.work + cmd.Env = envForDir(cmd.Dir, os.Environ()) + out, err := cmd.CombinedOutput() + return err == nil && !bytes.Contains(out, []byte("unrecognized")) +} + // gccArchArgs returns arguments to pass to gcc based on the architecture. func (b *builder) gccArchArgs() []string { switch goarch { @@ -2866,12 +3003,6 @@ func (b *builder) cflags(p *Package, def bool) (cppflags, cflags, cxxflags, ldfl var cgoRe = regexp.MustCompile(`[/\\:]`) -var ( - cgoLibGccFile string - cgoLibGccErr error - cgoLibGccFileOnce sync.Once -) - func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofiles, gccfiles, gxxfiles, mfiles []string) (outGo, outObj []string, err error) { cgoCPPFLAGS, cgoCFLAGS, cgoCXXFLAGS, cgoLDFLAGS := b.cflags(p, true) _, cgoexeCFLAGS, _, _ := b.cflags(p, false) @@ -2882,11 +3013,16 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi cgoLDFLAGS = append(cgoLDFLAGS, "-lobjc") } + if buildMSan && p.ImportPath != "runtime/cgo" { + cgoCFLAGS = append([]string{"-fsanitize=memory"}, cgoCFLAGS...) + cgoLDFLAGS = append([]string{"-fsanitize=memory"}, cgoLDFLAGS...) + } + // Allows including _cgo_export.h from .[ch] files in the package. cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", obj) // cgo - // TODO: CGOPKGPATH, CGO_FLAGS? + // TODO: CGO_FLAGS? gofiles := []string{obj + "_cgo_gotypes.go"} cfiles := []string{"_cgo_main.c", "_cgo_export.c"} for _, fn := range cgofiles { @@ -2902,7 +3038,7 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi if p.Standard && p.ImportPath == "runtime/cgo" { cgoflags = append(cgoflags, "-import_runtime_cgo=false") } - if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/cgo") { + if p.Standard && (p.ImportPath == "runtime/race" || p.ImportPath == "runtime/msan" || p.ImportPath == "runtime/cgo") { cgoflags = append(cgoflags, "-import_syscall=false") } @@ -2954,7 +3090,9 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi var linkobj []string var bareLDFLAGS []string - // filter out -lsomelib, -l somelib, *.{so,dll,dylib}, and (on Darwin) -framework X + // When linking relocatable objects, various flags need to be + // filtered out as they are inapplicable and can cause some linkers + // to fail. for i := 0; i < len(cgoLDFLAGS); i++ { f := cgoLDFLAGS[i] switch { @@ -2970,7 +3108,6 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi case strings.HasSuffix(f, ".dylib"), strings.HasSuffix(f, ".so"), strings.HasSuffix(f, ".dll"): - continue // Remove any -fsanitize=foo flags. // Otherwise the compiler driver thinks that we are doing final link // and links sanitizer runtime into the object file. But we are not doing @@ -2979,27 +3116,27 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi // See issue 8788 for details. case strings.HasPrefix(f, "-fsanitize="): continue + // runpath flags not applicable unless building a shared + // object or executable; see issue 12115 for details. This + // is necessary as Go currently does not offer a way to + // specify the set of LDFLAGS that only apply to shared + // objects. + case strings.HasPrefix(f, "-Wl,-rpath"): + if f == "-Wl,-rpath" || f == "-Wl,-rpath-link" { + // Skip following argument to -rpath* too. + i++ + } default: bareLDFLAGS = append(bareLDFLAGS, f) } } - cgoLibGccFileOnce.Do(func() { - cgoLibGccFile, cgoLibGccErr = b.libgcc(p) - }) - if cgoLibGccFile == "" && cgoLibGccErr != nil { - return nil, nil, err - } - var staticLibs []string if goos == "windows" { - // libmingw32 and libmingwex might also use libgcc, so libgcc must come last, - // and they also have some inter-dependencies, so must use linker groups. + // libmingw32 and libmingwex have some inter-dependencies, + // so must use linker groups. staticLibs = []string{"-Wl,--start-group", "-lmingwex", "-lmingw32", "-Wl,--end-group"} } - if cgoLibGccFile != "" { - staticLibs = append(staticLibs, cgoLibGccFile) - } cflags := stringList(cgoCPPFLAGS, cgoCFLAGS) for _, cfile := range cfiles { @@ -3045,7 +3182,7 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi linkobj = append(linkobj, p.SysoFiles...) dynobj := obj + "_cgo_.o" - pie := goarch == "arm" && (goos == "linux" || goos == "android") + pie := (goarch == "arm" && goos == "linux") || goos == "android" if pie { // we need to use -pie for Linux/ARM to get accurate imported sym cgoLDFLAGS = append(cgoLDFLAGS, "-pie") } @@ -3083,6 +3220,10 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, pcCFLAGS, pcLDFLAGS, cgofi } ldflags := stringList(bareLDFLAGS, "-Wl,-r", "-nostdlib", staticLibs) + if b.gccSupportsNoPie() { + ldflags = append(ldflags, "-no-pie") + } + // We are creating an object file, so we don't want a build ID. ldflags = b.disableBuildID(ldflags) @@ -3217,7 +3358,7 @@ func (b *builder) swigIntSize(obj string) (intsize string, err error) { return "$INTBITS", nil } src := filepath.Join(b.work, "swig_intsize.go") - if err = ioutil.WriteFile(src, []byte(swigIntSizeCode), 0644); err != nil { + if err = ioutil.WriteFile(src, []byte(swigIntSizeCode), 0666); err != nil { return } srcs := []string{src} @@ -3339,32 +3480,38 @@ func (q *actionQueue) pop() *action { return heap.Pop(q).(*action) } -func raceInit() { - if !buildRace { +func instrumentInit() { + if !buildRace && !buildMSan { return } + if buildRace && buildMSan { + fmt.Fprintf(os.Stderr, "go %s: may not use -race and -msan simultaneously", flag.Args()[0]) + os.Exit(2) + } if goarch != "amd64" || goos != "linux" && goos != "freebsd" && goos != "darwin" && goos != "windows" { - fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0]) + fmt.Fprintf(os.Stderr, "go %s: -race and -msan are only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0]) os.Exit(2) } - buildGcflags = append(buildGcflags, "-race") - buildLdflags = append(buildLdflags, "-race") + if !buildContext.CgoEnabled { + fmt.Fprintf(os.Stderr, "go %s: -race requires cgo; enable cgo by setting CGO_ENABLED=1\n", flag.Args()[0]) + os.Exit(2) + } + if buildRace { + buildGcflags = append(buildGcflags, "-race") + buildLdflags = append(buildLdflags, "-race") + } else { + buildGcflags = append(buildGcflags, "-msan") + buildLdflags = append(buildLdflags, "-msan") + } if buildContext.InstallSuffix != "" { buildContext.InstallSuffix += "_" } - buildContext.InstallSuffix += "race" - buildContext.BuildTags = append(buildContext.BuildTags, "race") -} -// defaultSuffix returns file extension used for command files in -// current os environment. -func defaultSuffix() string { - switch runtime.GOOS { - case "windows": - return ".bat" - case "plan9": - return ".rc" - default: - return ".bash" + if buildRace { + buildContext.InstallSuffix += "race" + buildContext.BuildTags = append(buildContext.BuildTags, "race") + } else { + buildContext.InstallSuffix += "msan" + buildContext.BuildTags = append(buildContext.BuildTags, "msan") } } diff --git a/libgo/go/cmd/go/discovery.go b/libgo/go/cmd/go/discovery.go index b9f4279..f6992e9 100644 --- a/libgo/go/cmd/go/discovery.go +++ b/libgo/go/cmd/go/discovery.go @@ -41,9 +41,9 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { d.Strict = false var t xml.Token for { - t, err = d.Token() + t, err = d.RawToken() if err != nil { - if err == io.EOF { + if err == io.EOF || len(imports) > 0 { err = nil } return diff --git a/libgo/go/cmd/go/doc.go b/libgo/go/cmd/go/doc.go index 4a07dfe..9b8b8df 100644 --- a/libgo/go/cmd/go/doc.go +++ b/libgo/go/cmd/go/doc.go @@ -32,12 +32,17 @@ which is schematically one of these: go doc <pkg> go doc <sym>[.<method>] - go doc [<pkg>].<sym>[.<method>] + go doc [<pkg>.]<sym>[.<method>] + go doc [<pkg>.][<sym>.]<method> -The first item in this list matched by the argument is the one whose -documentation is printed. (See the examples below.) For packages, the order of -scanning is determined lexically, but the GOROOT tree is always scanned before -GOPATH. +The first item in this list matched by the argument is the one whose documentation +is printed. (See the examples below.) However, if the argument starts with a capital +letter it is assumed to identify a symbol or method in the current directory. + +For packages, the order of scanning is determined lexically in breadth-first order. +That is, the package presented is the one that matches the search and is nearest +the root and lexically first at its level of the hierarchy. The GOROOT tree is +always scanned in its entirety before GOPATH. If there is no package specified or matched, the package in the current directory is selected, so "go doc Foo" shows the documentation for symbol Foo in @@ -85,6 +90,14 @@ Examples: go doc text/template new # Two arguments Show documentation for text/template's New function. + At least in the current tree, these invocations all print the + documentation for json.Decoder's Decode method: + + go doc json.Decoder.Decode + go doc json.decoder.decode + go doc json.decode + cd go/src/encoding/json; go doc decode + Flags: -c Respect case when matching symbols. diff --git a/libgo/go/cmd/go/env.go b/libgo/go/cmd/go/env.go index 600acca..24f6127 100644 --- a/libgo/go/cmd/go/env.go +++ b/libgo/go/cmd/go/env.go @@ -33,6 +33,11 @@ func mkEnv() []envVar { var b builder b.init() + vendorExpValue := "0" + if go15VendorExperiment { + vendorExpValue = "1" + } + env := []envVar{ {"GOARCH", goarch}, {"GOBIN", gobin}, @@ -44,7 +49,7 @@ func mkEnv() []envVar { {"GORACE", os.Getenv("GORACE")}, {"GOROOT", goroot}, {"GOTOOLDIR", toolDir}, - {"GO15VENDOREXPERIMENT", os.Getenv("GO15VENDOREXPERIMENT")}, + {"GO15VENDOREXPERIMENT", vendorExpValue}, // disable escape codes in clang errors {"TERM", "dumb"}, diff --git a/libgo/go/cmd/go/generate.go b/libgo/go/cmd/go/generate.go index efdc229..cb54018 100644 --- a/libgo/go/cmd/go/generate.go +++ b/libgo/go/cmd/go/generate.go @@ -22,7 +22,7 @@ import ( var cmdGenerate = &Command{ Run: runGenerate, - UsageLine: "generate [-run regexp] [file.go... | packages]", + UsageLine: "generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]", Short: "generate Go files by processing source", Long: ` Generate runs commands described by directives within existing @@ -115,12 +115,14 @@ Go generate accepts one specific flag: any trailing spaces and final newline) matches the expression. -It also accepts the standard build flags -v, -n, and -x. +It also accepts the standard build flags including -v, -n, and -x. The -v flag prints the names of packages and files as they are processed. The -n flag prints commands that would be executed. The -x flag prints commands as they are executed. +For more about build flags, see 'go help build'. + For more about specifying packages, see 'go help packages'. `, } @@ -179,6 +181,7 @@ type Generator struct { pkg string commands map[string][]string lineNum int // current line number. + env []string } // run runs the generators in the current file. @@ -242,6 +245,7 @@ func (g *Generator) run() (ok bool) { } } + g.setEnv() words := g.split(string(buf)) if len(words) == 0 { g.errorf("no arguments to directive") @@ -269,6 +273,19 @@ func isGoGenerate(buf []byte) bool { return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t")) } +// setEnv sets the extra environment variables used when executing a +// single go:generate command. +func (g *Generator) setEnv() { + g.env = []string{ + "GOARCH=" + runtime.GOARCH, + "GOOS=" + runtime.GOOS, + "GOFILE=" + g.file, + "GOLINE=" + strconv.Itoa(g.lineNum), + "GOPACKAGE=" + g.pkg, + "DOLLAR=" + "$", + } +} + // split breaks the line into words, evaluating quoted // strings and evaluating environment variables. // The initial //go:generate element is present in line. @@ -345,22 +362,13 @@ func (g *Generator) errorf(format string, args ...interface{}) { // expandVar expands the $XXX invocation in word. It is called // by os.Expand. func (g *Generator) expandVar(word string) string { - switch word { - case "GOARCH": - return buildContext.GOARCH - case "GOOS": - return buildContext.GOOS - case "GOFILE": - return g.file - case "GOLINE": - return fmt.Sprint(g.lineNum) - case "GOPACKAGE": - return g.pkg - case "DOLLAR": - return "$" - default: - return os.Getenv(word) + w := word + "=" + for _, e := range g.env { + if strings.HasPrefix(e, w) { + return e[len(w):] + } } + return os.Getenv(word) } // identLength returns the length of the identifier beginning the string. @@ -396,13 +404,7 @@ func (g *Generator) exec(words []string) { cmd.Stderr = os.Stderr // Run the command in the package directory. cmd.Dir = g.dir - env := []string{ - "GOARCH=" + runtime.GOARCH, - "GOOS=" + runtime.GOOS, - "GOFILE=" + g.file, - "GOPACKAGE=" + g.pkg, - } - cmd.Env = mergeEnvLists(env, origEnv) + cmd.Env = mergeEnvLists(g.env, origEnv) err := cmd.Run() if err != nil { g.errorf("running %q: %s", words[0], err) diff --git a/libgo/go/cmd/go/generate_test.go b/libgo/go/cmd/go/generate_test.go index 169d71c..ba06692 100644 --- a/libgo/go/cmd/go/generate_test.go +++ b/libgo/go/cmd/go/generate_test.go @@ -39,6 +39,7 @@ func TestGenerateCommandParse(t *testing.T) { pkg: "sys", commands: make(map[string][]string), } + g.setEnv() g.setShorthand([]string{"-command", "yacc", "go", "tool", "yacc"}) for _, test := range splitTests { // First with newlines. diff --git a/libgo/go/cmd/go/get.go b/libgo/go/cmd/go/get.go index e95201a..a298049 100644 --- a/libgo/go/cmd/go/get.go +++ b/libgo/go/cmd/go/get.go @@ -45,16 +45,22 @@ missing packages but does not use it to look for updates to existing packages. Get also accepts build flags to control the installation. See 'go help build'. +When checking out a new package, get creates the target directory +GOPATH/src/<import-path>. If the GOPATH contains multiple entries, +get uses the first one. See 'go help gopath'. + When checking out or updating a package, get looks for a branch or tag that matches the locally installed version of Go. The most important rule is that if the local installation is running version "go1", get searches for a branch or tag named "go1". If no such version exists it retrieves the most recent version of the package. -If the vendoring experiment is enabled (see 'go help gopath'), -then when go get checks out or updates a Git repository, +Unless vendoring support is disabled (see 'go help gopath'), +when go get checks out or updates a Git repository, it also updates any git submodules referenced by the repository. +Get never checks out or updates code stored in vendor directories. + For more about specifying packages, see 'go help packages'. For more about how 'go get' finds source code to @@ -84,8 +90,12 @@ func runGet(cmd *Command, args []string) { // Disable any prompting for passwords by Git. // Only has an effect for 2.3.0 or later, but avoiding // the prompt in earlier versions is just too hard. - // See golang.org/issue/9341. - os.Setenv("GIT_TERMINAL_PROMPT", "0") + // If user has explicitly set GIT_TERMINAL_PROMPT=1, keep + // prompting. + // See golang.org/issue/9341 and golang.org/issue/12706. + if os.Getenv("GIT_TERMINAL_PROMPT") == "" { + os.Setenv("GIT_TERMINAL_PROMPT", "0") + } // Phase 1. Download/update. var stk importStack diff --git a/libgo/go/cmd/go/go_test.go b/libgo/go/cmd/go/go_test.go index 77b2628..a901ca8 100644 --- a/libgo/go/cmd/go/go_test.go +++ b/libgo/go/cmd/go/go_test.go @@ -31,8 +31,7 @@ var ( exeSuffix string // ".exe" on Windows - builder = testenv.Builder() - skipExternalBuilder = false // skip external tests on this builder + skipExternal = false // skip external tests ) func init() { @@ -44,14 +43,21 @@ func init() { case "arm", "arm64": canRun = false } - } - - if strings.HasPrefix(builder+"-", "freebsd-arm-") { - skipExternalBuilder = true - canRun = false - } - - switch runtime.GOOS { + case "linux": + switch runtime.GOARCH { + case "arm": + // many linux/arm machines are too slow to run + // the full set of external tests. + skipExternal = true + } + case "freebsd": + switch runtime.GOARCH { + case "arm": + // many freebsd/arm machines are too slow to run + // the full set of external tests. + skipExternal = true + canRun = false + } case "windows": exeSuffix = ".exe" } @@ -83,8 +89,6 @@ func TestMain(m *testing.M) { case "linux", "darwin", "freebsd", "windows": canRace = canCgo && runtime.GOARCH == "amd64" } - - measureTick("./testgo" + exeSuffix) } // Don't let these environment variables confuse the test. @@ -103,24 +107,8 @@ func TestMain(m *testing.M) { // The length of an mtime tick on this system. This is an estimate of // how long we need to sleep to ensure that the mtime of two files is // different. -var mtimeTick time.Duration - -// measureTick sets mtimeTick by looking at the rounding of the mtime -// of a file. -func measureTick(path string) { - st, err := os.Stat(path) - if err != nil { - // Default to one second, the most conservative value. - mtimeTick = time.Second - return - } - mtime := st.ModTime() - t := time.Microsecond - for mtime.Round(t).Equal(mtime) && t < time.Second { - t *= 10 - } - mtimeTick = t -} +// We used to try to be clever but that didn't always work (see golang.org/issue/12205). +var mtimeTick time.Duration = 1 * time.Second // Manage a single run of the testgo binary. type testgoData struct { @@ -138,8 +126,8 @@ type testgoData struct { func testgo(t *testing.T) *testgoData { testenv.MustHaveGoBuild(t) - if skipExternalBuilder { - t.Skip("skipping external tests on %s builder", builder) + if skipExternal { + t.Skip("skipping external tests on %s/%s", runtime.GOOS, runtime.GOARCH) } return &testgoData{t: t} @@ -452,7 +440,7 @@ func (tg *testgoData) grepCountBoth(match string) int { // removed if it exists. func (tg *testgoData) creatingTemp(path string) { if filepath.IsAbs(path) && !strings.HasPrefix(path, tg.tempdir) { - tg.t.Fatal("internal testsuite error: creatingTemp(%q) with absolute path not in temporary directory", path) + tg.t.Fatalf("internal testsuite error: creatingTemp(%q) with absolute path not in temporary directory", path) } // If we have changed the working directory, make sure we have // an absolute path, because we are going to change directory @@ -671,17 +659,48 @@ func TestGoBuildDashAInDevBranch(t *testing.T) { tg.grepStderr("runtime", "testgo build -a math in dev branch DID NOT build runtime, but should have") } -func TestGoBuilDashAInReleaseBranch(t *testing.T) { +func TestGoBuildDashAInReleaseBranch(t *testing.T) { if testing.Short() { t.Skip("don't rebuild the standard library in short mode") } tg := testgo(t) defer tg.cleanup() - tg.run("install", "math") // should be up to date already but just in case + tg.run("install", "math", "net/http") // should be up to date already but just in case tg.setenv("TESTGO_IS_GO_RELEASE", "1") - tg.run("build", "-v", "-a", "math") - tg.grepStderr("runtime", "testgo build -a math in dev branch did not build runtime, but should have") + tg.run("install", "-v", "-a", "math") + tg.grepStderr("runtime", "testgo build -a math in release branch DID NOT build runtime, but should have") + + // Now runtime.a is updated (newer mtime), so everything would look stale if not for being a release. + // + tg.run("build", "-v", "net/http") + tg.grepStderrNot("strconv", "testgo build -v net/http in release branch with newer runtime.a DID build strconv but should not have") + tg.grepStderrNot("golang.org/x/net/http2/hpack", "testgo build -v net/http in release branch with newer runtime.a DID build .../golang.org/x/net/http2/hpack but should not have") + tg.grepStderrNot("net/http", "testgo build -v net/http in release branch with newer runtime.a DID build net/http but should not have") +} + +func TestGoListStandard(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.cd(runtime.GOROOT() + "/src") + tg.run("list", "-f", "{{if not .Standard}}{{.ImportPath}}{{end}}", "./...") + stdout := tg.getStdout() + for _, line := range strings.Split(stdout, "\n") { + if strings.HasPrefix(line, "_/") && strings.HasSuffix(line, "/src") { + // $GOROOT/src shows up if there are any .go files there. + // We don't care. + continue + } + if line == "" { + continue + } + t.Errorf("package in GOROOT not listed as standard: %v", line) + } + + // Similarly, expanding std should include some of our vendored code. + tg.run("list", "std", "cmd") + tg.grepStdout("golang.org/x/net/http2/hpack", "list std cmd did not mention vendored hpack") + tg.grepStdout("golang.org/x/arch/x86/x86asm", "list std cmd did not mention vendored x86asm") } func TestGoInstallCleansUpAfterGoBuild(t *testing.T) { @@ -775,6 +794,28 @@ func TestGoInstallDetectsRemovedFiles(t *testing.T) { tg.wantStale("mypkg", "./testgo list mypkg claims mypkg is NOT stale after removing y.go; should be stale") } +func TestWildcardMatchesSyntaxErrorDirs(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("src/mypkg/x.go", `package mypkg`) + tg.tempFile("src/mypkg/y.go", `pkg mypackage`) + tg.setenv("GOPATH", tg.path(".")) + tg.cd(tg.path("src/mypkg")) + tg.runFail("list", "./...") + tg.runFail("build", "./...") + tg.runFail("install", "./...") +} + +func TestGoListWithTags(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("src/mypkg/x.go", "// +build thetag\n\npackage mypkg\n") + tg.setenv("GOPATH", tg.path(".")) + tg.cd(tg.path("./src")) + tg.run("list", "-tags=thetag", "./my...") + tg.grepStdout("mypkg", "did not find mypkg") +} + func TestGoInstallErrorOnCrossCompileToBin(t *testing.T) { if testing.Short() { t.Skip("don't install into GOROOT in short mode") @@ -951,6 +992,16 @@ func TestInternalPackagesOutsideGOROOTAreRespected(t *testing.T) { tg.grepBoth("use of internal package not allowed", "wrote error message for testdata/testinternal2") } +func TestRunInternal(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + dir := filepath.Join(tg.pwd(), "testdata") + tg.setenv("GOPATH", dir) + tg.run("run", filepath.Join(dir, "src/run/good.go")) + tg.runFail("run", filepath.Join(dir, "src/run/bad.go")) + tg.grepStderr("use of internal package not allowed", "unexpected error for run/bad.go") +} + func testMove(t *testing.T, vcs, url, base, config string) { testenv.MustHaveExternalNetwork(t) @@ -1053,7 +1104,6 @@ func TestImportCommentConflict(t *testing.T) { // cmd/go: custom import path checking should not apply to github.com/xxx/yyy. func TestIssue10952(t *testing.T) { testenv.MustHaveExternalNetwork(t) - if _, err := exec.LookPath("git"); err != nil { t.Skip("skipping because git binary not found") } @@ -1071,6 +1121,34 @@ func TestIssue10952(t *testing.T) { tg.run("get", "-d", "-u", importPath) } +func TestGetGitDefaultBranch(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + if _, err := exec.LookPath("git"); err != nil { + t.Skip("skipping because git binary not found") + } + + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempDir("src") + tg.setenv("GOPATH", tg.path(".")) + + // This repo has two branches, master and another-branch. + // The another-branch is the default that you get from 'git clone'. + // The go get command variants should not override this. + const importPath = "github.com/rsc/go-get-default-branch" + + tg.run("get", "-d", importPath) + repoDir := tg.path("src/" + importPath) + defer tg.resetReadOnlyFlagAll(repoDir) + tg.runGit(repoDir, "branch", "--contains", "HEAD") + tg.grepStdout(`\* another-branch`, "not on correct default branch") + + tg.run("get", "-d", "-u", importPath) + tg.runGit(repoDir, "branch", "--contains", "HEAD") + tg.grepStdout(`\* another-branch`, "not on correct default branch") +} + func TestDisallowedCSourceFiles(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -1139,6 +1217,15 @@ func TestInstallFailsWithNoBuildableFiles(t *testing.T) { tg.grepStderr("no buildable Go source files", "go install cgotest did not report 'no buildable Go Source files'") } +func TestRelativeGOBINFail(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("triv.go", `package main; func main() {}`) + tg.setenv("GOBIN", ".") + tg.runFail("install") + tg.grepStderr("cannot install, GOBIN must be an absolute path", "go install must fail if $GOBIN is a relative path") +} + // Test that without $GOBIN set, binaries get installed // into the GOPATH bin directory. func TestInstallIntoGOPATH(t *testing.T) { @@ -1150,9 +1237,21 @@ func TestInstallIntoGOPATH(t *testing.T) { tg.wantExecutable("testdata/bin/go-cmd-test"+exeSuffix, "go install go-cmd-test did not write to testdata/bin/go-cmd-test") } +// Issue 12407 +func TestBuildOutputToDevNull(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping because /dev/null is a regular file on plan9") + } + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.run("build", "-o", os.DevNull, "go-cmd-test") +} + func TestPackageMainTestImportsArchiveNotBinary(t *testing.T) { tg := testgo(t) defer tg.cleanup() + tg.parallel() gobin := filepath.Join(tg.pwd(), "testdata", "bin") tg.creatingTemp(gobin) tg.setenv("GOBIN", gobin) @@ -1165,6 +1264,17 @@ func TestPackageMainTestImportsArchiveNotBinary(t *testing.T) { tg.run("test", "main_test") } +// Issue 12690 +func TestPackageNotStaleWithTrailingSlash(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + goroot := runtime.GOROOT() + tg.setenv("GOROOT", goroot+"/") + tg.wantNotStale("runtime", "with trailing slash in GOROOT, runtime listed as stale") + tg.wantNotStale("os", "with trailing slash in GOROOT, os listed as stale") + tg.wantNotStale("io", "with trailing slash in GOROOT, io listed as stale") +} + // With $GOBIN set, binaries get installed to $GOBIN. func TestInstallIntoGOBIN(t *testing.T) { tg := testgo(t) @@ -1357,6 +1467,18 @@ func TestGoListCmdOnlyShowsCommands(t *testing.T) { } } +func TestGoListDedupsPackages(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.run("list", "xtestonly", "./testdata/src/xtestonly/...") + got := strings.TrimSpace(tg.getStdout()) + const want = "xtestonly" + if got != want { + t.Errorf("got %q; want %q", got, want) + } +} + // Issue 4096. Validate the output of unsuccessful go install foo/quxx. func TestUnsuccessfulGoInstallShouldMentionMissingPackage(t *testing.T) { tg := testgo(t) @@ -1833,6 +1955,9 @@ func TestIssue6480(t *testing.T) { // cmd/cgo: undefined reference when linking a C-library using gccgo func TestIssue7573(t *testing.T) { + if !canCgo { + t.Skip("skipping because cgo not enabled") + } if _, err := exec.LookPath("gccgo"); err != nil { t.Skip("skipping because no gccgo compiler found") } @@ -1931,6 +2056,13 @@ func TestGoTestFooTestWorks(t *testing.T) { tg.run("test", "testdata/standalone_test.go") } +func TestGoTestFlagsAfterPackage(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.run("test", "testdata/flag_test.go", "-v", "-args", "-v=7") // Two distinct -v flags. + tg.run("test", "-v", "testdata/flag_test.go", "-args", "-v=7") // Two distinct -v flags. +} + func TestGoTestXtestonlyWorks(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -1991,6 +2123,21 @@ func TestGoGenerateRunFlag(t *testing.T) { tg.grepStdoutNot("no", "go generate -run yes ./testdata/generate/test4.go selected no") } +func TestGoGenerateEnv(t *testing.T) { + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("skipping because %s does not have the env command", runtime.GOOS) + } + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.tempFile("env.go", "package main\n\n//go:generate env") + tg.run("generate", tg.path("env.go")) + for _, v := range []string{"GOARCH", "GOOS", "GOFILE", "GOLINE", "GOPACKAGE", "DOLLAR"} { + tg.grepStdout("^"+v+"=", "go generate environment missing "+v) + } +} + func TestGoGetCustomDomainWildcard(t *testing.T) { testenv.MustHaveExternalNetwork(t) @@ -2051,6 +2198,17 @@ func TestGoGetRscIoToolstash(t *testing.T) { tg.run("get", "./toolstash") } +// Issue 13037: Was not parsing <meta> tags in 404 served over HTTPS +func TestGoGetHTTPS404(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.tempDir("src") + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "bazil.org/fuse/fs/fstestutil") +} + // Test that you can not import a main package. func TestIssue4210(t *testing.T) { tg := testgo(t) @@ -2128,7 +2286,11 @@ func TestGoGetInsecureCustomDomain(t *testing.T) { } func TestIssue10193(t *testing.T) { + t.Skip("depends on code.google.com") testenv.MustHaveExternalNetwork(t) + if _, err := exec.LookPath("hg"); err != nil { + t.Skip("skipping because hg binary not found") + } tg := testgo(t) defer tg.cleanup() @@ -2199,7 +2361,7 @@ func TestGoTestImportErrorStack(t *testing.T) { tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) tg.runFail("test", "testdep/p1") if !strings.Contains(tg.stderr.String(), out) { - t.Fatal("did not give full import stack:\n\n%s", tg.stderr.String()) + t.Fatalf("did not give full import stack:\n\n%s", tg.stderr.String()) } } @@ -2387,3 +2549,12 @@ func TestGoBuildARM(t *testing.T) { tg.run("build", "hello.go") tg.grepStderrNot("unable to find math.a", "did not build math.a correctly") } + +func TestIssue13655(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + for _, pkg := range []string{"runtime", "runtime/internal/atomic"} { + tg.run("list", "-f", "{{.Deps}}", pkg) + tg.grepStdout("runtime/internal/sys", "did not find required dependency of "+pkg+" on runtime/internal/sys") + } +} diff --git a/libgo/go/cmd/go/go_unix_test.go b/libgo/go/cmd/go/go_unix_test.go new file mode 100644 index 0000000..0d85859e --- /dev/null +++ b/libgo/go/cmd/go/go_unix_test.go @@ -0,0 +1,31 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package main_test + +import ( + "os" + "syscall" + "testing" +) + +func TestGoBuildUmask(t *testing.T) { + // Do not use tg.parallel; avoid other tests seeing umask manipulation. + mask := syscall.Umask(0077) // prohibit low bits + defer syscall.Umask(mask) + tg := testgo(t) + defer tg.cleanup() + tg.tempFile("x.go", `package main; func main() {}`) + tg.creatingTemp("x") + tg.run("build", tg.path("x.go")) + fi, err := os.Stat("x") + if err != nil { + t.Fatal(err) + } + if mode := fi.Mode(); mode&0077 != 0 { + t.Fatalf("wrote x with mode=%v, wanted no 0077 bits", mode) + } +} diff --git a/libgo/go/cmd/go/help.go b/libgo/go/cmd/go/help.go index 5dff267..d8e7efe 100644 --- a/libgo/go/cmd/go/help.go +++ b/libgo/go/cmd/go/help.go @@ -79,6 +79,14 @@ internally at Google all begin with 'google', and paths denoting remote repositories begin with the path to the code, such as 'github.com/user/repo'. +Packages in a program need not have unique package names, +but there are two reserved package names with special meaning. +The name main indicates a command, not a library. +Commands are built into binaries and cannot be imported. +The name documentation indicates documentation for +a non-Go program in the directory. Files in package documentation +are ignored by the go command. + As a special case, if the package list is a list of .go files from a single directory, the command is applied to a single synthesized package made up of exactly those files, ignoring any build constraints @@ -261,10 +269,10 @@ unless it is being referred to by that import path. In this way, import comments let package authors make sure the custom import path is used and not a direct path to the underlying code hosting site. -If the vendoring experiment is enabled (see 'go help gopath'), -then import path checking is disabled for code found within vendor trees. -This makes it possible to copy code into alternate locations in vendor trees -without needing to update import comments. +If vendoring is enabled (see 'go help gopath'), then import path checking is +disabled for code found within vendor trees. This makes it possible to copy +code into alternate locations in vendor trees without needing to update import +comments. See https://golang.org/s/go14customimport for details. `, @@ -307,7 +315,7 @@ DIR/bin/quux, not DIR/bin/foo/quux. The "foo/" prefix is stripped so that you can add DIR/bin to your PATH to get at the installed commands. If the GOBIN environment variable is set, commands are installed to the directory it names instead -of DIR/bin. +of DIR/bin. GOBIN must be an absolute path. Here's an example directory layout: @@ -365,13 +373,10 @@ See https://golang.org/s/go14internal for details. Vendor Directories -Go 1.5 includes experimental support for using local copies -of external dependencies to satisfy imports of those dependencies, -often referred to as vendoring. Setting the environment variable -GO15VENDOREXPERIMENT=1 enables that experimental support. +Go 1.6 includes support for using local copies of external dependencies +to satisfy imports of those dependencies, often referred to as vendoring. -When the vendor experiment is enabled, -code below a directory named "vendor" is importable only +Code below a directory named "vendor" is importable only by code in the directory tree rooted at the parent of "vendor", and only using an import path that omits the prefix up to and including the vendor element. @@ -409,12 +414,18 @@ top-level "crash/bang". Code in vendor directories is not subject to import path checking (see 'go help importpath'). -When the vendor experiment is enabled, 'go get' checks out -submodules when checking out or updating a git repository -(see 'go help get'). +When 'go get' checks out or updates a git repository, it now also +updates submodules. + +Vendor directories do not affect the placement of new repositories +being checked out for the first time by 'go get': those are always +placed in the main GOPATH, never in a vendor subtree. -The vendoring semantics are an experiment, and they may change -in future releases. Once settled, they will be on by default. +In Go 1.5, as an experiment, setting the environment variable +GO15VENDOREXPERIMENT=1 enabled these features. +As of Go 1.6 they are on by default. To turn them off, set +GO15VENDOREXPERIMENT=0. In Go 1.7, the environment +variable will stop having any effect. See https://golang.org/s/go15vendor for details. `, @@ -487,7 +498,7 @@ Special-purpose environment variables: File names in stack traces are rewritten from GOROOT to GOROOT_FINAL. GO15VENDOREXPERIMENT - Set to 1 to enable the Go 1.5 vendoring experiment. + Set to 0 to disable vendoring semantics. GO_EXTLINK_ENABLED Whether the linker should use external linking mode when using -linkmode=auto with code that uses cgo. @@ -570,5 +581,10 @@ are: -buildmode=exe Build the listed main packages and everything they import into executables. Packages not named main are ignored. + + -buildmode=pie + Build the listed main packages and everything they import into + position independent executables (PIE). Packages not named + main are ignored. `, } diff --git a/libgo/go/cmd/go/http.go b/libgo/go/cmd/go/http.go index 7979c41..3a6f19d 100644 --- a/libgo/go/cmd/go/http.go +++ b/libgo/go/cmd/go/http.go @@ -12,6 +12,7 @@ package main import ( + "crypto/tls" "fmt" "io" "io/ioutil" @@ -24,8 +25,17 @@ import ( // httpClient is the default HTTP client, but a variable so it can be // changed by tests, without modifying http.DefaultClient. var httpClient = http.DefaultClient -var impatientHTTPClient = &http.Client{ + +// impatientInsecureHTTPClient is used in -insecure mode, +// when we're connecting to https servers that might not be there +// or might be using self-signed certificates. +var impatientInsecureHTTPClient = &http.Client{ Timeout: time.Duration(5 * time.Second), + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, } type httpError struct { @@ -71,7 +81,7 @@ func httpsOrHTTP(importPath string, security securityMode) (urlStr string, body log.Printf("Fetching %s", urlStr) } if security == insecure && scheme == "https" { // fail earlier - res, err = impatientHTTPClient.Get(urlStr) + res, err = impatientInsecureHTTPClient.Get(urlStr) } else { res, err = httpClient.Get(urlStr) } @@ -83,16 +93,12 @@ func httpsOrHTTP(importPath string, security securityMode) (urlStr string, body } } urlStr, res, err := fetch("https") - if err != nil || res.StatusCode != 200 { + if err != nil { if buildV { - if err != nil { - log.Printf("https fetch failed.") - } else { - log.Printf("ignoring https fetch with status code %d", res.StatusCode) - } + log.Printf("https fetch failed: %v", err) } - closeBody(res) if security == insecure { + closeBody(res) urlStr, res, err = fetch("http") } } diff --git a/libgo/go/cmd/go/list.go b/libgo/go/cmd/go/list.go index 35c7cc4..8f741a6 100644 --- a/libgo/go/cmd/go/list.go +++ b/libgo/go/cmd/go/list.go @@ -78,6 +78,14 @@ syntax of package template. The default output is equivalent to -f XTestImports []string // imports from XTestGoFiles } +The error information, if any, is + + type PackageError struct { + ImportStack []string // shortest path from package named on command line to this one + Pos string // position of error (if present, file:line:col) + Err string // the error itself + } + The template function "join" calls strings.Join. The template function "context" returns the build context, defined as: diff --git a/libgo/go/cmd/go/main.go b/libgo/go/cmd/go/main.go index 8ebde89..c8697ff 100644 --- a/libgo/go/cmd/go/main.go +++ b/libgo/go/cmd/go/main.go @@ -285,8 +285,8 @@ func printUsage(w io.Writer) { func usage() { // special case "go test -h" if len(os.Args) > 1 && os.Args[1] == "test" { - os.Stdout.WriteString(testUsage + "\n\n" + - strings.TrimSpace(testFlag1) + "\n\n" + + os.Stderr.WriteString(testUsage + "\n\n" + + strings.TrimSpace(testFlag1) + "\n\n\t" + strings.TrimSpace(testFlag2) + "\n") os.Exit(2) } @@ -353,7 +353,7 @@ func importPathsNoDotExpansion(args []string) []string { } else { a = path.Clean(a) } - if a == "all" || a == "std" || a == "cmd" { + if isMetaPackage(a) { out = append(out, allPackages(a)...) continue } @@ -554,7 +554,7 @@ func allPackages(pattern string) []string { func matchPackages(pattern string) []string { match := func(string) bool { return true } treeCanMatch := func(string) bool { return true } - if pattern != "all" && pattern != "std" && pattern != "cmd" { + if !isMetaPackage(pattern) { match = matchPattern(pattern) treeCanMatch = treeCanMatchPattern(pattern) } @@ -588,10 +588,9 @@ func matchPackages(pattern string) []string { } name := filepath.ToSlash(path[len(src):]) - if pattern == "std" && (strings.Contains(name, ".") || name == "cmd") { + if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") { // The name "std" is only the standard library. - // If the name has a dot, assume it's a domain name for go get, - // and if the name is cmd, it's the root of the command tree. + // If the name is cmd, it's the root of the command tree. return filepath.SkipDir } if !treeCanMatch(name) { @@ -674,7 +673,14 @@ func matchPackagesInFS(pattern string) []string { if !match(name) { return nil } - if _, err = build.ImportDir(path, 0); err != nil { + + // We keep the directory if we can import it, or if we can't import it + // due to invalid Go source files. This means that directories containing + // parse errors will be built (and fail) instead of being silently skipped + // as not matching the pattern. Go 1.5 and earlier skipped, but that + // behavior means people miss serious mistakes. + // See golang.org/issue/11407. + if p, err := buildContext.ImportDir(path, 0); err != nil && (p == nil || len(p.InvalidGoFiles) == 0) { if _, noGo := err.(*build.NoGoError); !noGo { log.Print(err) } diff --git a/libgo/go/cmd/go/note.go b/libgo/go/cmd/go/note.go index 97e1865..f846eeb 100644 --- a/libgo/go/cmd/go/note.go +++ b/libgo/go/cmd/go/note.go @@ -7,6 +7,7 @@ package main import ( "bytes" "debug/elf" + "debug/macho" "encoding/binary" "fmt" "io" @@ -69,11 +70,11 @@ func readELFNote(filename, name string, typ int32) ([]byte, error) { var elfGoNote = []byte("Go\x00\x00") -// readELFGoBuildID the Go build ID string from an ELF binary. -// The Go build ID is stored in a note described by an ELF PT_NOTE prog header. -// The caller has already opened filename, to get f, and read the first 4 kB out, in data. +// The Go build ID is stored in a note described by an ELF PT_NOTE prog +// header. The caller has already opened filename, to get f, and read +// at least 4 kB out, in data. func readELFGoBuildID(filename string, f *os.File, data []byte) (buildid string, err error) { - // Assume the note content is in the first 4 kB, already read. + // Assume the note content is in the data, already read. // Rewrite the ELF header to set shnum to 0, so that we can pass // the data to elf.NewFile and it will decode the Prog list but not // try to read the section headers and the string table from disk. @@ -95,22 +96,92 @@ func readELFGoBuildID(filename string, f *os.File, data []byte) (buildid string, return "", &os.PathError{Path: filename, Op: "parse", Err: err} } for _, p := range ef.Progs { - if p.Type != elf.PT_NOTE || p.Off >= uint64(len(data)) || p.Off+p.Filesz >= uint64(len(data)) || p.Filesz < 16 { + if p.Type != elf.PT_NOTE || p.Filesz < 16 { continue } - note := data[p.Off : p.Off+p.Filesz] - nameSize := ef.ByteOrder.Uint32(note) - valSize := ef.ByteOrder.Uint32(note[4:]) - tag := ef.ByteOrder.Uint32(note[8:]) - name := note[12:16] - if nameSize != 4 || 16+valSize > uint32(len(note)) || tag != elfGoBuildIDTag || !bytes.Equal(name, elfGoNote) { - continue + var note []byte + if p.Off+p.Filesz < uint64(len(data)) { + note = data[p.Off : p.Off+p.Filesz] + } else { + // For some linkers, such as the Solaris linker, + // the buildid may not be found in data (which + // likely contains the first 16kB of the file) + // or even the first few megabytes of the file + // due to differences in note segment placement; + // in that case, extract the note data manually. + _, err = f.Seek(int64(p.Off), 0) + if err != nil { + return "", err + } + + note = make([]byte, p.Filesz) + _, err = io.ReadFull(f, note) + if err != nil { + return "", err + } } - return string(note[16 : 16+valSize]), nil + filesz := p.Filesz + for filesz >= 16 { + nameSize := ef.ByteOrder.Uint32(note) + valSize := ef.ByteOrder.Uint32(note[4:]) + tag := ef.ByteOrder.Uint32(note[8:]) + name := note[12:16] + if nameSize == 4 && 16+valSize <= uint32(len(note)) && tag == elfGoBuildIDTag && bytes.Equal(name, elfGoNote) { + return string(note[16 : 16+valSize]), nil + } + + nameSize = (nameSize + 3) &^ 3 + valSize = (valSize + 3) &^ 3 + notesz := uint64(12 + nameSize + valSize) + if filesz <= notesz { + break + } + filesz -= notesz + note = note[notesz:] + } } // No note. Treat as successful but build ID empty. return "", nil } + +// The Go build ID is stored at the beginning of the Mach-O __text segment. +// The caller has already opened filename, to get f, and read a few kB out, in data. +// Sadly, that's not guaranteed to hold the note, because there is an arbitrary amount +// of other junk placed in the file ahead of the main text. +func readMachoGoBuildID(filename string, f *os.File, data []byte) (buildid string, err error) { + // If the data we want has already been read, don't worry about Mach-O parsing. + // This is both an optimization and a hedge against the Mach-O parsing failing + // in the future due to, for example, the name of the __text section changing. + if b, err := readRawGoBuildID(filename, data); b != "" && err == nil { + return b, err + } + + mf, err := macho.NewFile(f) + if err != nil { + return "", &os.PathError{Path: filename, Op: "parse", Err: err} + } + + sect := mf.Section("__text") + if sect == nil { + // Every binary has a __text section. Something is wrong. + return "", &os.PathError{Path: filename, Op: "parse", Err: fmt.Errorf("cannot find __text section")} + } + + // It should be in the first few bytes, but read a lot just in case, + // especially given our past problems on OS X with the build ID moving. + // There shouldn't be much difference between reading 4kB and 32kB: + // the hard part is getting to the data, not transferring it. + n := sect.Size + if n > uint64(BuildIDReadSize) { + n = uint64(BuildIDReadSize) + } + buf := make([]byte, n) + if _, err := f.ReadAt(buf, int64(sect.Offset)); err != nil { + return "", err + } + + return readRawGoBuildID(filename, buf) +} diff --git a/libgo/go/cmd/go/note_test.go b/libgo/go/cmd/go/note_test.go index 3d64451..811734b 100644 --- a/libgo/go/cmd/go/note_test.go +++ b/libgo/go/cmd/go/note_test.go @@ -6,11 +6,29 @@ package main_test import ( main "cmd/go" + "go/build" "runtime" "testing" ) func TestNoteReading(t *testing.T) { + testNoteReading(t) +} + +func TestNoteReading2K(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("2kB is not enough on %s", runtime.GOOS) + } + // Set BuildIDReadSize to 2kB to exercise Mach-O parsing more strictly. + defer func(old int) { + main.BuildIDReadSize = old + }(main.BuildIDReadSize) + main.BuildIDReadSize = 2 * 1024 + + testNoteReading(t) +} + +func testNoteReading(t *testing.T) { tg := testgo(t) defer tg.cleanup() tg.tempFile("hello.go", `package main; func main() { print("hello, world\n") }`) @@ -24,26 +42,25 @@ func TestNoteReading(t *testing.T) { t.Fatalf("buildID in hello binary = %q, want %q", id, buildID) } - if runtime.GOOS == "linux" && runtime.GOARCH == "ppc64le" { - t.Skipf("skipping - golang.org/issue/11184") + switch { + case !build.Default.CgoEnabled: + t.Skipf("skipping - no cgo, so assuming external linking not available") + case runtime.GOOS == "linux" && (runtime.GOARCH == "ppc64le" || runtime.GOARCH == "ppc64"): + t.Skipf("skipping - external linking not supported, golang.org/issue/11184") + case runtime.GOOS == "linux" && (runtime.GOARCH == "mips64le" || runtime.GOARCH == "mips64"): + t.Skipf("skipping - external linking not supported, golang.org/issue/12560") + case runtime.GOOS == "openbsd" && runtime.GOARCH == "arm": + t.Skipf("skipping - external linking not supported, golang.org/issue/10619") + case runtime.GOOS == "plan9": + t.Skipf("skipping - external linking not supported") } - switch runtime.GOOS { - case "plan9": - // no external linking - t.Logf("no external linking - skipping linkmode=external test") - - case "solaris": - t.Logf("skipping - golang.org/issue/12178") - - default: - tg.run("build", "-ldflags", "-buildid="+buildID+" -linkmode=external", "-o", tg.path("hello.exe"), tg.path("hello.go")) - id, err := main.ReadBuildIDFromBinary(tg.path("hello.exe")) - if err != nil { - t.Fatalf("reading build ID from hello binary (linkmode=external): %v", err) - } - if id != buildID { - t.Fatalf("buildID in hello binary = %q, want %q (linkmode=external)", id, buildID) - } + tg.run("build", "-ldflags", "-buildid="+buildID+" -linkmode=external", "-o", tg.path("hello.exe"), tg.path("hello.go")) + id, err = main.ReadBuildIDFromBinary(tg.path("hello.exe")) + if err != nil { + t.Fatalf("reading build ID from hello binary (linkmode=external): %v", err) + } + if id != buildID { + t.Fatalf("buildID in hello binary = %q, want %q (linkmode=external)", id, buildID) } } diff --git a/libgo/go/cmd/go/pkg.go b/libgo/go/cmd/go/pkg.go index 3270a8b..73ea206 100644 --- a/libgo/go/cmd/go/pkg.go +++ b/libgo/go/cmd/go/pkg.go @@ -118,7 +118,7 @@ func (p *Package) vendored(imports []string) []string { seen := make(map[string]bool) var all []string for _, path := range imports { - path, _ = vendoredImportPath(p, path) + path = vendoredImportPath(p, path) if !seen[path] { seen[path] = true all = append(all, path) @@ -156,7 +156,7 @@ func (p *Package) copyBuild(pp *build.Package) { if buildContext.Compiler == "gccgo" { p.Standard = stdpkg[p.ImportPath] } else { - p.Standard = p.Goroot && p.ImportPath != "" && !strings.Contains(p.ImportPath, ".") + p.Standard = p.Goroot && p.ImportPath != "" && isStandardImportPath(p.ImportPath) } p.GoFiles = pp.GoFiles p.CgoFiles = pp.CgoFiles @@ -181,6 +181,19 @@ func (p *Package) copyBuild(pp *build.Package) { p.XTestImports = pp.XTestImports } +// isStandardImportPath reports whether $GOROOT/src/path should be considered +// part of the standard distribution. For historical reasons we allow people to add +// their own code to $GOROOT instead of using $GOPATH, but we assume that +// code will start with a domain name (dot in the first element). +func isStandardImportPath(path string) bool { + i := strings.Index(path, "/") + if i < 0 { + i = len(path) + } + elem := path[:i] + return !strings.Contains(elem, ".") +} + // A PackageError describes an error loading information about a package. type PackageError struct { ImportStack []string // shortest path from package named on command line to this one @@ -254,11 +267,14 @@ func reloadPackage(arg string, stk *importStack) *Package { return loadPackage(arg, stk) } -// The Go 1.5 vendoring experiment is enabled by setting GO15VENDOREXPERIMENT=1. +// The Go 1.5 vendoring experiment was enabled by setting GO15VENDOREXPERIMENT=1. +// In Go 1.6 this is on by default and is disabled by setting GO15VENDOREXPERIMENT=0. +// In Go 1.7 the variable will stop having any effect. // The variable is obnoxiously long so that years from now when people find it in // their profiles and wonder what it does, there is some chance that a web search // might answer the question. -var go15VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") == "1" +// There is a copy of this variable in src/go/build/build.go. Delete that one when this one goes away. +var go15VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") != "0" // dirToImportPath returns the pseudo-import path we use for a package // outside the Go path. It begins with _/ and then contains the full path @@ -314,11 +330,14 @@ func loadImport(path, srcDir string, parent *Package, stk *importStack, importPo importPath := path origPath := path isLocal := build.IsLocalImport(path) - var vendorSearch []string if isLocal { importPath = dirToImportPath(filepath.Join(srcDir, path)) } else if mode&useVendor != 0 { - path, vendorSearch = vendoredImportPath(parent, path) + // We do our own vendor resolution, because we want to + // find out the key to use in packageCache without the + // overhead of repeated calls to buildContext.Import. + // The code is also needed in a few other places anyway. + path = vendoredImportPath(parent, path) importPath = path } @@ -345,29 +364,12 @@ func loadImport(path, srcDir string, parent *Package, stk *importStack, importPo // // TODO: After Go 1, decide when to pass build.AllowBinary here. // See issue 3268 for mistakes to avoid. - bp, err := buildContext.Import(path, srcDir, build.ImportComment) - - // If we got an error from go/build about package not found, - // it contains the directories from $GOROOT and $GOPATH that - // were searched. Add to that message the vendor directories - // that were searched. - if err != nil && len(vendorSearch) > 0 { - // NOTE(rsc): The direct text manipulation here is fairly awful, - // but it avoids defining new go/build API (an exported error type) - // late in the Go 1.5 release cycle. If this turns out to be a more general - // problem we could define a real error type when the decision can be - // considered more carefully. - text := err.Error() - if strings.Contains(text, "cannot find package \"") && strings.Contains(text, "\" in any of:\n\t") { - old := strings.SplitAfter(text, "\n") - lines := []string{old[0]} - for _, dir := range vendorSearch { - lines = append(lines, "\t"+dir+" (vendor tree)\n") - } - lines = append(lines, old[1:]...) - err = errors.New(strings.Join(lines, "")) - } + buildMode := build.ImportComment + if !go15VendorExperiment || mode&useVendor == 0 || path != origPath { + // Not vendoring, or we already found the vendored path. + buildMode |= build.IgnoreVendor } + bp, err := buildContext.Import(path, srcDir, buildMode) bp.ImportPath = importPath if gobin != "" { bp.BinDir = gobin @@ -377,7 +379,7 @@ func loadImport(path, srcDir string, parent *Package, stk *importStack, importPo err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment) } p.load(stk, bp, err) - if p.Error != nil && len(importPos) > 0 { + if p.Error != nil && p.Error.Pos == "" && len(importPos) > 0 { pos := importPos[0] pos.Filename = shortPath(pos.Filename) p.Error.Pos = pos.String() @@ -411,14 +413,11 @@ func isDir(path string) bool { // vendoredImportPath returns the expansion of path when it appears in parent. // If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, -// x/vendor/path, vendor/path, or else stay x/y/z if none of those exist. +// x/vendor/path, vendor/path, or else stay path if none of those exist. // vendoredImportPath returns the expanded path or, if no expansion is found, the original. -// If no expansion is found, vendoredImportPath also returns a list of vendor directories -// it searched along the way, to help prepare a useful error message should path turn -// out not to exist. -func vendoredImportPath(parent *Package, path string) (found string, searched []string) { +func vendoredImportPath(parent *Package, path string) (found string) { if parent == nil || parent.Root == "" || !go15VendorExperiment { - return path, nil + return path } dir := filepath.Clean(parent.Dir) root := filepath.Join(parent.Root, "src") @@ -438,7 +437,7 @@ func vendoredImportPath(parent *Package, path string) (found string, searched [] continue } targ := filepath.Join(dir[:i], vpath) - if isDir(targ) { + if isDir(targ) && hasGoFiles(targ) { // We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy. // We know the import path for parent's dir. // We chopped off some number of path elements and @@ -453,14 +452,26 @@ func vendoredImportPath(parent *Package, path string) (found string, searched [] // and found c:\gopath\src\vendor\path. // We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7). // Use "vendor/path" without any prefix. - return vpath, nil + return vpath } - return parent.ImportPath[:len(parent.ImportPath)-chopped] + "/" + vpath, nil + return parent.ImportPath[:len(parent.ImportPath)-chopped] + "/" + vpath } - // Note the existence of a vendor directory in case path is not found anywhere. - searched = append(searched, targ) } - return path, searched + return path +} + +// hasGoFiles reports whether dir contains any files with names ending in .go. +// For a vendor check we must exclude directories that contain no .go files. +// Otherwise it is not possible to vendor just a/b/c and still import the +// non-vendored a/b. See golang.org/issue/13832. +func hasGoFiles(dir string) bool { + fis, _ := ioutil.ReadDir(dir) + for _, fi := range fis { + if !fi.IsDir() && strings.HasSuffix(fi.Name(), ".go") { + return true + } + } + return false } // reusePackage reuses package p to satisfy the import at the top @@ -522,7 +533,7 @@ func disallowInternal(srcDir string, p *Package, stk *importStack) *Package { i-- // rewind over slash in ".../internal" } parent := p.Dir[:i+len(p.Dir)-len(p.ImportPath)] - if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) { + if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { return p } @@ -619,7 +630,7 @@ func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Pack return p } parent := p.Dir[:truncateTo] - if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) { + if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { return p } @@ -712,6 +723,7 @@ func expandScanner(err error) error { var raceExclude = map[string]bool{ "runtime/race": true, + "runtime/msan": true, "runtime/cgo": true, "cmd/cgo": true, "syscall": true, @@ -725,6 +737,7 @@ var cgoExclude = map[string]bool{ var cgoSyscallExclude = map[string]bool{ "runtime/cgo": true, "runtime/race": true, + "runtime/msan": true, } // load populates p using information from bp, err, which should @@ -829,27 +842,40 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package importPaths = append(importPaths, "syscall") } - // Currently build mode c-shared, or -linkshared, forces + // Currently build modes c-shared, pie, and -linkshared force // external linking mode, and external linking mode forces an // import of runtime/cgo. - if p.Name == "main" && !p.Goroot && (buildBuildmode == "c-shared" || buildLinkshared) { + if p.Name == "main" && !p.Goroot && (buildBuildmode == "c-shared" || buildBuildmode == "pie" || buildLinkshared) { importPaths = append(importPaths, "runtime/cgo") } - // Everything depends on runtime, except runtime and unsafe. - if !p.Standard || (p.ImportPath != "runtime" && p.ImportPath != "unsafe") { + // Everything depends on runtime, except runtime, its internal + // subpackages, and unsafe. + if !p.Standard || (p.ImportPath != "runtime" && !strings.HasPrefix(p.ImportPath, "runtime/internal/") && p.ImportPath != "unsafe") { importPaths = append(importPaths, "runtime") // When race detection enabled everything depends on runtime/race. // Exclude certain packages to avoid circular dependencies. if buildRace && (!p.Standard || !raceExclude[p.ImportPath]) { importPaths = append(importPaths, "runtime/race") } + // MSan uses runtime/msan. + if buildMSan && (!p.Standard || !raceExclude[p.ImportPath]) { + importPaths = append(importPaths, "runtime/msan") + } // On ARM with GOARM=5, everything depends on math for the link. if p.Name == "main" && goarch == "arm" { importPaths = append(importPaths, "math") } } + // Runtime and its internal packages depend on runtime/internal/sys, + // so that they pick up the generated zversion.go file. + // This can be an issue particularly for runtime/internal/atomic; + // see issue 13655. + if p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal/")) && p.ImportPath != "runtime/internal/sys" { + importPaths = append(importPaths, "runtime/internal/sys") + } + // Build list of full paths to all Go files in the package, // for use by commands like go fmt. p.gofiles = stringList(p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles) @@ -931,6 +957,17 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package } } } + if p.Standard && !p1.Standard && p.Error == nil { + p.Error = &PackageError{ + ImportStack: stk.copy(), + Err: fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath), + } + pos := p.build.ImportPos[path] + if len(pos) > 0 { + p.Error.Pos = pos[0].String() + } + } + path = p1.ImportPath importPaths[i] = path if i < len(p.Imports) { @@ -1613,15 +1650,24 @@ func packagesAndErrors(args []string) []*Package { } args = importPaths(args) - var pkgs []*Package - var stk importStack - var set = make(map[string]bool) + var ( + pkgs []*Package + stk importStack + seenArg = make(map[string]bool) + seenPkg = make(map[*Package]bool) + ) for _, arg := range args { - if !set[arg] { - pkgs = append(pkgs, loadPackage(arg, &stk)) - set[arg] = true + if seenArg[arg] { + continue + } + seenArg[arg] = true + pkg := loadPackage(arg, &stk) + if seenPkg[pkg] { + continue } + seenPkg[pkg] = true + pkgs = append(pkgs, pkg) } computeStale(pkgs...) @@ -1792,8 +1838,17 @@ var ( goBuildEnd = []byte("\"\n \xff") elfPrefix = []byte("\x7fELF") + + machoPrefixes = [][]byte{ + {0xfe, 0xed, 0xfa, 0xce}, + {0xfe, 0xed, 0xfa, 0xcf}, + {0xce, 0xfa, 0xed, 0xfe}, + {0xcf, 0xfa, 0xed, 0xfe}, + } ) +var BuildIDReadSize = 32 * 1024 // changed for testing + // ReadBuildIDFromBinary reads the build ID from a binary. // // ELF binaries store the build ID in a proper PT_NOTE section. @@ -1808,10 +1863,11 @@ func ReadBuildIDFromBinary(filename string) (id string, err error) { return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown} } - // Read the first 16 kB of the binary file. + // Read the first 32 kB of the binary file. // That should be enough to find the build ID. // In ELF files, the build ID is in the leading headers, - // which are typically less than 4 kB, not to mention 16 kB. + // which are typically less than 4 kB, not to mention 32 kB. + // In Mach-O files, there's no limit, so we have to parse the file. // On other systems, we're trying to read enough that // we get the beginning of the text segment in the read. // The offset where the text segment begins in a hello @@ -1819,7 +1875,6 @@ func ReadBuildIDFromBinary(filename string) (id string, err error) { // // Plan 9: 0x20 // Windows: 0x600 - // Mach-O: 0x2000 // f, err := os.Open(filename) if err != nil { @@ -1827,7 +1882,7 @@ func ReadBuildIDFromBinary(filename string) (id string, err error) { } defer f.Close() - data := make([]byte, 16*1024) + data := make([]byte, BuildIDReadSize) _, err = io.ReadFull(f, data) if err == io.ErrUnexpectedEOF { err = nil @@ -1839,7 +1894,17 @@ func ReadBuildIDFromBinary(filename string) (id string, err error) { if bytes.HasPrefix(data, elfPrefix) { return readELFGoBuildID(filename, f, data) } + for _, m := range machoPrefixes { + if bytes.HasPrefix(data, m) { + return readMachoGoBuildID(filename, f, data) + } + } + + return readRawGoBuildID(filename, data) +} +// readRawGoBuildID finds the raw build ID stored in text segment data. +func readRawGoBuildID(filename string, data []byte) (id string, err error) { i := bytes.Index(data, goBuildPrefix) if i < 0 { // Missing. Treat as successful but build ID empty. diff --git a/libgo/go/cmd/go/pkg_test.go b/libgo/go/cmd/go/pkg_test.go index 06b9f0a..1e7ca2c 100644 --- a/libgo/go/cmd/go/pkg_test.go +++ b/libgo/go/cmd/go/pkg_test.go @@ -5,6 +5,9 @@ package main import ( + "io/ioutil" + "os" + "path/filepath" "reflect" "strings" "testing" @@ -57,6 +60,15 @@ var parseMetaGoImportsTests = []struct { <body>`, []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, }, + { + `<!doctype html><meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`, + []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, + }, + { + // XML doesn't like <div style=position:relative>. + `<!doctype html><title>Page Not Found</title><meta name=go-import content="chitin.io/chitin git https://github.com/chitin-io/chitin"><div style=position:relative>DRAFT</div>`, + []metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin"}}, + }, } func TestParseMetaGoImports(t *testing.T) { @@ -71,3 +83,109 @@ func TestParseMetaGoImports(t *testing.T) { } } } + +func TestSharedLibName(t *testing.T) { + // TODO(avdva) - make these values platform-specific + prefix := "lib" + suffix := ".so" + testData := []struct { + args []string + pkgs []*Package + expected string + expectErr bool + rootedAt string + }{ + { + args: []string{"std"}, + pkgs: []*Package{}, + expected: "std", + }, + { + args: []string{"std", "cmd"}, + pkgs: []*Package{}, + expected: "std,cmd", + }, + { + args: []string{}, + pkgs: []*Package{&Package{ImportPath: "gopkg.in/somelib"}}, + expected: "gopkg.in-somelib", + }, + { + args: []string{"./..."}, + pkgs: []*Package{&Package{ImportPath: "somelib"}}, + expected: "somelib", + rootedAt: "somelib", + }, + { + args: []string{"../somelib", "../somelib"}, + pkgs: []*Package{&Package{ImportPath: "somelib"}}, + expected: "somelib", + }, + { + args: []string{"../lib1", "../lib2"}, + pkgs: []*Package{&Package{ImportPath: "gopkg.in/lib1"}, &Package{ImportPath: "gopkg.in/lib2"}}, + expected: "gopkg.in-lib1,gopkg.in-lib2", + }, + { + args: []string{"./..."}, + pkgs: []*Package{ + &Package{ImportPath: "gopkg.in/dir/lib1"}, + &Package{ImportPath: "gopkg.in/lib2"}, + &Package{ImportPath: "gopkg.in/lib3"}, + }, + expected: "gopkg.in", + rootedAt: "gopkg.in", + }, + { + args: []string{"std", "../lib2"}, + pkgs: []*Package{}, + expectErr: true, + }, + { + args: []string{"all", "./"}, + pkgs: []*Package{}, + expectErr: true, + }, + { + args: []string{"cmd", "fmt"}, + pkgs: []*Package{}, + expectErr: true, + }, + } + for _, data := range testData { + func() { + if data.rootedAt != "" { + tmpGopath, err := ioutil.TempDir("", "gopath") + if err != nil { + t.Fatal(err) + } + oldGopath := buildContext.GOPATH + defer func() { + os.RemoveAll(tmpGopath) + buildContext.GOPATH = oldGopath + os.Chdir(cwd) + }() + root := filepath.Join(tmpGopath, "src", data.rootedAt) + err = os.MkdirAll(root, 0755) + if err != nil { + t.Fatal(err) + } + buildContext.GOPATH = tmpGopath + os.Chdir(root) + } + computed, err := libname(data.args, data.pkgs) + if err != nil { + if !data.expectErr { + t.Errorf("libname returned an error %q, expected a name", err.Error()) + } + } else if data.expectErr { + t.Errorf("libname returned %q, expected an error", computed) + } else { + expected := prefix + data.expected + suffix + if expected != computed { + t.Errorf("libname returned %q, expected %q", computed, expected) + } + } + }() + } +} diff --git a/libgo/go/cmd/go/run.go b/libgo/go/cmd/go/run.go index f6da373..bf10f4f 100644 --- a/libgo/go/cmd/go/run.go +++ b/libgo/go/cmd/go/run.go @@ -64,7 +64,7 @@ func printStderr(args ...interface{}) (int, error) { } func runRun(cmd *Command, args []string) { - raceInit() + instrumentInit() buildModeInit() var b builder b.init() @@ -89,8 +89,18 @@ func runRun(cmd *Command, args []string) { fatalf("%s", p.Error) } p.omitDWARF = true - for _, err := range p.DepsErrors { - errorf("%s", err) + if len(p.DepsErrors) > 0 { + // Since these are errors in dependencies, + // the same error might show up multiple times, + // once in each package that depends on it. + // Only print each once. + printed := map[*PackageError]bool{} + for _, err := range p.DepsErrors { + if !printed[err] { + printed[err] = true + errorf("%s", err) + } + } } exitIfErrors() if p.Name != "main" { diff --git a/libgo/go/cmd/go/test.go b/libgo/go/cmd/go/test.go index aadfdf6..1cf5892 100644 --- a/libgo/go/cmd/go/test.go +++ b/libgo/go/cmd/go/test.go @@ -13,7 +13,6 @@ import ( "go/doc" "go/parser" "go/token" - "log" "os" "os/exec" "path" @@ -33,7 +32,7 @@ func init() { cmdTest.Run = runTest } -const testUsage = "test [-c] [-i] [build and test flags] [packages] [flags for test binary]" +const testUsage = "test [build/test flags] [packages] [build/test flags & test binary flags]" var cmdTest = &Command{ CustomFlags: true, @@ -68,11 +67,6 @@ non-test installation. ` + strings.TrimSpace(testFlag1) + ` See 'go help testflag' for details. -If the test binary needs any other flags, they should be presented after the -package names. The go tool treats as a flag the first argument that begins with -a minus sign that it does not recognize itself; that argument and all subsequent -arguments are passed as arguments to the test binary. - For more about build flags, see 'go help build'. For more about specifying packages, see 'go help packages'. @@ -83,10 +77,16 @@ See also: go build, go vet. const testFlag1 = ` In addition to the build flags, the flags handled by 'go test' itself are: + -args + Pass the remainder of the command line (everything after -args) + to the test binary, uninterpreted and unchanged. + Because this flag consumes the remainder of the command line, + the package list (if present) must appear before this flag. + -c - Compile the test binary to pkg.test but do not run it - (where pkg is the last element of the package's import path). - The file name can be changed with the -o flag. + Compile the test binary to pkg.test but do not run it + (where pkg is the last element of the package's import path). + The file name can be changed with the -o flag. -exec xprog Run the test binary using xprog. The behavior is the same as @@ -97,8 +97,8 @@ In addition to the build flags, the flags handled by 'go test' itself are: Do not run the test. -o file - Compile the test binary to the named file. - The test still runs (unless -c or -i is specified). + Compile the test binary to the named file. + The test still runs (unless -c or -i is specified). The test binary also accepts flags that control execution of the test; these flags are also accessible by 'go test'. @@ -207,6 +207,10 @@ const testFlag2 = ` Allow parallel execution of test functions that call t.Parallel. The value of this flag is the maximum number of tests to run simultaneously; by default, it is set to the value of GOMAXPROCS. + Note that -parallel only applies within a single test binary. + The 'go test' command may run tests for different packages + in parallel as well, according to the setting of the -p flag + (see 'go help build'). -run regexp Run only those tests and examples matching the regular @@ -230,25 +234,63 @@ const testFlag2 = ` Verbose output: log all tests as they are run. Also print all text from Log and Logf calls even if the test succeeds. -The test binary, called pkg.test where pkg is the name of the -directory containing the package sources, can be invoked directly -after building it with 'go test -c'. When invoking the test binary -directly, each of the standard flag names must be prefixed with 'test.', -as in -test.run=TestMyFunc or -test.v. +Each of these flags is also recognized with an optional 'test.' prefix, +as in -test.v. When invoking the generated test binary (the result of +'go test -c') directly, however, the prefix is mandatory. + +The 'go test' command rewrites or removes recognized flags, +as appropriate, both before and after the optional package list, +before invoking the test binary. -When running 'go test', flags not listed above are passed through -unaltered. For instance, the command +For instance, the command - go test -x -v -cpuprofile=prof.out -dir=testdata -update + go test -v -myflag testdata -cpuprofile=prof.out -x will compile the test binary and then run it as - pkg.test -test.v -test.cpuprofile=prof.out -dir=testdata -update + pkg.test -test.v -myflag testdata -test.cpuprofile=prof.out + +(The -x flag is removed because it applies only to the go command's +execution, not to the test itself.) The test flags that generate profiles (other than for coverage) also leave the test binary in pkg.test for use when analyzing the profiles. -Flags not recognized by 'go test' must be placed after any specified packages. +When 'go test' runs a test binary, it does so from within the +corresponding package's source code directory. Depending on the test, +it may be necessary to do the same when invoking a generated test +binary directly. + +The command-line package list, if present, must appear before any +flag not known to the go test command. Continuing the example above, +the package list would have to appear before -myflag, but could appear +on either side of -v. + +To keep an argument for a test binary from being interpreted as a +known flag or a package name, use -args (see 'go help test') which +passes the remainder of the command line through to the test binary +uninterpreted and unaltered. + +For instance, the command + + go test -v -args -x -v + +will compile the test binary and then run it as + + pkg.test -test.v -x -v + +Similarly, + + go test -args math + +will compile the test binary and then run it as + + pkg.test math + +In the first example, the -x and the second -v are passed through to the +test binary unchanged and with no effect on the go command itself. +In the second example, the argument math is passed through to the test +binary, instead of being interpreted as the package list. ` var helpTestfunc = &Command{ @@ -328,7 +370,7 @@ func runTest(cmd *Command, args []string) { findExecCmd() // initialize cached result - raceInit() + instrumentInit() buildModeInit() pkgs := packagesForBuild(pkgArgs) if len(pkgs) == 0 { @@ -396,7 +438,7 @@ func runTest(cmd *Command, args []string) { if deps["C"] { delete(deps, "C") deps["runtime/cgo"] = true - if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace { + if goos == runtime.GOOS && goarch == runtime.GOARCH && !buildRace && !buildMSan { deps["cmd/cgo"] = true } } @@ -442,7 +484,7 @@ func runTest(cmd *Command, args []string) { } for _, p := range testCoverPkgs { if !used[p.ImportPath] { - log.Printf("warning: no packages being tested depend on %s", p.ImportPath) + fmt.Fprintf(os.Stderr, "warning: no packages being tested depend on %s\n", p.ImportPath) } } @@ -547,6 +589,9 @@ func runTest(cmd *Command, args []string) { if buildRace { extraOpts = "-race " } + if buildMSan { + extraOpts = "-msan " + } fmt.Fprintf(os.Stderr, "installing these packages with 'go test %s-i%s' will speed future tests.\n\n", extraOpts, args) } @@ -829,10 +874,12 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, } } - // writeTestmain writes _testmain.go. This must happen after recompileForTest, - // because recompileForTest modifies XXX. - if err := writeTestmain(filepath.Join(testDir, "_testmain.go"), t); err != nil { - return nil, nil, nil, err + if !buildN { + // writeTestmain writes _testmain.go. This must happen after recompileForTest, + // because recompileForTest modifies XXX. + if err := writeTestmain(filepath.Join(testDir, "_testmain.go"), t); err != nil { + return nil, nil, nil, err + } } computeStale(pmain) diff --git a/libgo/go/cmd/go/testdata/flag_test.go b/libgo/go/cmd/go/testdata/flag_test.go new file mode 100644 index 0000000..ddf613d --- /dev/null +++ b/libgo/go/cmd/go/testdata/flag_test.go @@ -0,0 +1,16 @@ +package flag_test + +import ( + "flag" + "log" + "testing" +) + +var v = flag.Int("v", 0, "v flag") + +// Run this as go test pkg -v=7 +func TestVFlagIsSet(t *testing.T) { + if *v != 7 { + log.Fatal("v flag not set") + } +} diff --git a/libgo/go/cmd/go/testdata/src/run/bad.go b/libgo/go/cmd/go/testdata/src/run/bad.go new file mode 100644 index 0000000..c1cc3ac --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/run/bad.go @@ -0,0 +1,5 @@ +package main + +import _ "run/subdir/internal/private" + +func main() {} diff --git a/libgo/go/cmd/go/testdata/src/run/good.go b/libgo/go/cmd/go/testdata/src/run/good.go new file mode 100644 index 0000000..0b67dce --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/run/good.go @@ -0,0 +1,5 @@ +package main + +import _ "run/internal" + +func main() {} diff --git a/libgo/go/cmd/go/testdata/src/run/internal/internal.go b/libgo/go/cmd/go/testdata/src/run/internal/internal.go new file mode 100644 index 0000000..5bf0569 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/run/internal/internal.go @@ -0,0 +1 @@ +package internal diff --git a/libgo/go/cmd/go/testdata/src/run/subdir/internal/private/private.go b/libgo/go/cmd/go/testdata/src/run/subdir/internal/private/private.go new file mode 100644 index 0000000..735e4dc --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/run/subdir/internal/private/private.go @@ -0,0 +1 @@ +package private diff --git a/libgo/go/cmd/go/testdata/src/vend/dir1/dir1.go b/libgo/go/cmd/go/testdata/src/vend/dir1/dir1.go new file mode 100644 index 0000000..b719ead --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/dir1/dir1.go @@ -0,0 +1 @@ +package dir1 diff --git a/libgo/go/cmd/go/testdata/src/vend/hello/hello_test.go b/libgo/go/cmd/go/testdata/src/vend/hello/hello_test.go index 5e72ada..7190f59 100644 --- a/libgo/go/cmd/go/testdata/src/vend/hello/hello_test.go +++ b/libgo/go/cmd/go/testdata/src/vend/hello/hello_test.go @@ -7,6 +7,6 @@ import ( func TestMsgInternal(t *testing.T) { if strings.Msg != "hello, world" { - t.Fatal("unexpected msg: %v", strings.Msg) + t.Fatalf("unexpected msg: %v", strings.Msg) } } diff --git a/libgo/go/cmd/go/testdata/src/vend/hello/hellox_test.go b/libgo/go/cmd/go/testdata/src/vend/hello/hellox_test.go index 96e6049..3f2165b 100644 --- a/libgo/go/cmd/go/testdata/src/vend/hello/hellox_test.go +++ b/libgo/go/cmd/go/testdata/src/vend/hello/hellox_test.go @@ -7,6 +7,6 @@ import ( func TestMsgExternal(t *testing.T) { if strings.Msg != "hello, world" { - t.Fatal("unexpected msg: %v", strings.Msg) + t.Fatalf("unexpected msg: %v", strings.Msg) } } diff --git a/libgo/go/cmd/go/testdata/src/vend/vendor/vend/dir1/dir2/dir2.go b/libgo/go/cmd/go/testdata/src/vend/vendor/vend/dir1/dir2/dir2.go new file mode 100644 index 0000000..6fe35e9 --- /dev/null +++ b/libgo/go/cmd/go/testdata/src/vend/vendor/vend/dir1/dir2/dir2.go @@ -0,0 +1 @@ +package dir2 diff --git a/libgo/go/cmd/go/testdata/src/vend/x/x.go b/libgo/go/cmd/go/testdata/src/vend/x/x.go index ae526eb..bdcde57 100644 --- a/libgo/go/cmd/go/testdata/src/vend/x/x.go +++ b/libgo/go/cmd/go/testdata/src/vend/x/x.go @@ -3,3 +3,5 @@ package x import _ "p" import _ "q" import _ "r" +import _ "vend/dir1" // not vendored +import _ "vend/dir1/dir2" // vendored diff --git a/libgo/go/cmd/go/testflag.go b/libgo/go/cmd/go/testflag.go index 1f3e3d3..873df1f 100644 --- a/libgo/go/cmd/go/testflag.go +++ b/libgo/go/cmd/go/testflag.go @@ -87,6 +87,7 @@ func init() { func testFlags(args []string) (packageNames, passToTest []string) { inPkg := false outputDir := "" + var explicitArgs []string for i := 0; i < len(args); i++ { if !strings.HasPrefix(args[i], "-") { if !inPkg && packageNames == nil { @@ -114,6 +115,12 @@ func testFlags(args []string) (packageNames, passToTest []string) { // make non-nil: we have seen the empty package list packageNames = []string{} } + if args[i] == "-args" || args[i] == "--args" { + // -args or --args signals that everything that follows + // should be passed to the test. + explicitArgs = args[i+1:] + break + } passToTest = append(passToTest, args[i]) continue } @@ -191,6 +198,8 @@ func testFlags(args []string) (packageNames, passToTest []string) { } passToTest = append(passToTest, "-test.outputdir", dir) } + + passToTest = append(passToTest, explicitArgs...) return } diff --git a/libgo/go/cmd/go/tool.go b/libgo/go/cmd/go/tool.go index 4b94939..9a552d6 100644 --- a/libgo/go/cmd/go/tool.go +++ b/libgo/go/cmd/go/tool.go @@ -104,6 +104,7 @@ func runTool(cmd *Command, args []string) { fmt.Printf("%s\n", cmd) return } + args[0] = toolPath // in case the tool wants to re-exec itself, e.g. cmd/dist toolCmd := &exec.Cmd{ Path: toolPath, Args: args, diff --git a/libgo/go/cmd/go/vcs.go b/libgo/go/cmd/go/vcs.go index 28a7540..074dd8b 100644 --- a/libgo/go/cmd/go/vcs.go +++ b/libgo/go/cmd/go/vcs.go @@ -122,7 +122,7 @@ var vcsGit = &vcsCmd{ name: "Git", cmd: "git", - createCmd: []string{"clone {repo} {dir}", "--git-dir={dir}/.git submodule update --init --recursive"}, + createCmd: []string{"clone {repo} {dir}", "-C {dir} submodule update --init --recursive"}, downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, tagCmd: []tagCmd{ @@ -137,8 +137,9 @@ var vcsGit = &vcsCmd{ // both createCmd and downloadCmd update the working dir. // No need to do more here. We used to 'checkout master' // but that doesn't work if the default branch is not named master. + // DO NOT add 'checkout master' here. // See golang.org/issue/9032. - tagSyncDefault: []string{"checkout master", "submodule update --init --recursive"}, + tagSyncDefault: []string{"submodule update --init --recursive"}, scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, pingCmd: "ls-remote {scheme}://{repo}", @@ -385,9 +386,6 @@ func (v *vcsCmd) create(dir, repo string) error { // download downloads any new changes for the repo in dir. func (v *vcsCmd) download(dir string) error { - if err := v.fixDetachedHead(dir); err != nil { - return err - } for _, cmd := range v.downloadCmd { if !go15VendorExperiment && strings.Contains(cmd, "submodule") { continue @@ -399,30 +397,6 @@ func (v *vcsCmd) download(dir string) error { return nil } -// fixDetachedHead switches a Git repository in dir from a detached head to the master branch. -// Go versions before 1.2 downloaded Git repositories in an unfortunate way -// that resulted in the working tree state being on a detached head. -// That meant the repository was not usable for normal Git operations. -// Go 1.2 fixed that, but we can't pull into a detached head, so if this is -// a Git repository we check for being on a detached head and switch to the -// real branch, almost always called "master". -// TODO(dsymonds): Consider removing this for Go 1.3. -func (v *vcsCmd) fixDetachedHead(dir string) error { - if v != vcsGit { - return nil - } - - // "git symbolic-ref HEAD" succeeds iff we are not on a detached head. - if err := v.runVerboseOnly(dir, "symbolic-ref HEAD"); err == nil { - // not on a detached head - return nil - } - if buildV { - log.Printf("%s on detached head; repairing", dir) - } - return v.run(dir, "checkout master") -} - // tags returns the list of available tags for the repo in dir. func (v *vcsCmd) tags(dir string) ([]string, error) { var tags []string @@ -567,16 +541,8 @@ func repoRootForImportPath(importPath string, security securityMode) (*repoRoot, lookup = lookup[:i] } rr, err = repoRootForImportDynamic(lookup, security) - - // repoRootForImportDynamic returns error detail - // that is irrelevant if the user didn't intend to use a - // dynamic import in the first place. - // Squelch it. if err != nil { - if buildV { - log.Printf("import %q: %v", importPath, err) - } - err = fmt.Errorf("unrecognized import path %q", importPath) + err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err) } } if err != nil { @@ -891,7 +857,7 @@ var vcsPaths = []*vcsPath{ // General syntax for any server. // Must be last. { - re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`, + re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`, ping: true, }, } diff --git a/libgo/go/cmd/go/vcs_test.go b/libgo/go/cmd/go/vcs_test.go index f5d5e4f..a90c206 100644 --- a/libgo/go/cmd/go/vcs_test.go +++ b/libgo/go/cmd/go/vcs_test.go @@ -18,14 +18,14 @@ func TestRepoRootForImportPath(t *testing.T) { path string want *repoRoot }{ - { + /*{ "code.google.com/p/go", &repoRoot{ vcs: vcsHg, repo: "https://code.google.com/p/go", }, }, - /*{ + { "code.google.com/r/go", &repoRoot{ vcs: vcsHg, diff --git a/libgo/go/cmd/go/vendor_test.go b/libgo/go/cmd/go/vendor_test.go index 1e8cf9c..006a8c9 100644 --- a/libgo/go/cmd/go/vendor_test.go +++ b/libgo/go/cmd/go/vendor_test.go @@ -24,12 +24,14 @@ func TestVendorImports(t *testing.T) { tg.run("list", "-f", "{{.ImportPath}} {{.Imports}}", "vend/...") want := ` vend [vend/vendor/p r] + vend/dir1 [] vend/hello [fmt vend/vendor/strings] vend/subdir [vend/vendor/p r] vend/vendor/p [] vend/vendor/q [] vend/vendor/strings [] - vend/x [vend/x/vendor/p vend/vendor/q vend/x/vendor/r] + vend/vendor/vend/dir1/dir2 [] + vend/x [vend/x/vendor/p vend/vendor/q vend/x/vendor/r vend/dir1 vend/vendor/vend/dir1/dir2] vend/x/invalid [vend/x/invalid/vendor/foo] vend/x/vendor/p [] vend/x/vendor/p/p [notfound] @@ -45,6 +47,14 @@ func TestVendorImports(t *testing.T) { } } +func TestVendorBuild(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.run("build", "vend/x") +} + func TestVendorRun(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -102,7 +112,7 @@ func TestVendorImportError(t *testing.T) { re := regexp.MustCompile(`cannot find package "notfound" in any of: .*[\\/]testdata[\\/]src[\\/]vend[\\/]x[\\/]vendor[\\/]notfound \(vendor tree\) - .*[\\/]testdata[\\/]src[\\/]vend[\\/]vendor[\\/]notfound \(vendor tree\) + .*[\\/]testdata[\\/]src[\\/]vend[\\/]vendor[\\/]notfound .*[\\/]src[\\/]notfound \(from \$GOROOT\) .*[\\/]testdata[\\/]src[\\/]notfound \(from \$GOPATH\)`) @@ -187,6 +197,18 @@ func TestVendorGetUpdate(t *testing.T) { tg.run("get", "-u", "github.com/rsc/go-get-issue-11864") } +func TestGetSubmodules(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.run("get", "-d", "github.com/rsc/go-get-issue-12612") + tg.run("get", "-u", "-d", "github.com/rsc/go-get-issue-12612") +} + func TestVendorCache(t *testing.T) { tg := testgo(t) defer tg.cleanup() diff --git a/libgo/go/cmd/gofmt/gofmt.go b/libgo/go/cmd/gofmt/gofmt.go index b2805ac..cfebeff 100644 --- a/libgo/go/cmd/gofmt/gofmt.go +++ b/libgo/go/cmd/gofmt/gofmt.go @@ -13,7 +13,6 @@ import ( "go/printer" "go/scanner" "go/token" - "internal/format" "io" "io/ioutil" "os" @@ -88,7 +87,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error return err } - file, sourceAdj, indentAdj, err := format.Parse(fileSet, filename, src, stdin) + file, sourceAdj, indentAdj, err := parse(fileSet, filename, src, stdin) if err != nil { return err } @@ -107,7 +106,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error simplify(file) } - res, err := format.Format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth}) + res, err := format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth}) if err != nil { return err } diff --git a/libgo/go/cmd/gofmt/internal.go b/libgo/go/cmd/gofmt/internal.go new file mode 100644 index 0000000..f764b10 --- /dev/null +++ b/libgo/go/cmd/gofmt/internal.go @@ -0,0 +1,176 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO(gri): This file and the file src/go/format/internal.go are +// the same (but for this comment and the package name). Do not modify +// one without the other. Determine if we can factor out functionality +// in a public API. See also #11844 for context. + +package main + +import ( + "bytes" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "strings" +) + +// parse parses src, which was read from the named file, +// as a Go source file, declaration, or statement list. +func parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) ( + file *ast.File, + sourceAdj func(src []byte, indent int) []byte, + indentAdj int, + err error, +) { + // Try as whole source file. + file, err = parser.ParseFile(fset, filename, src, parserMode) + // If there's no error, return. If the error is that the source file didn't begin with a + // package line and source fragments are ok, fall through to + // try as a source fragment. Stop and return on any other error. + if err == nil || !fragmentOk || !strings.Contains(err.Error(), "expected 'package'") { + return + } + + // If this is a declaration list, make it a source file + // by inserting a package clause. + // Insert using a ;, not a newline, so that the line numbers + // in psrc match the ones in src. + psrc := append([]byte("package p;"), src...) + file, err = parser.ParseFile(fset, filename, psrc, parserMode) + if err == nil { + sourceAdj = func(src []byte, indent int) []byte { + // Remove the package clause. + // Gofmt has turned the ; into a \n. + src = src[indent+len("package p\n"):] + return bytes.TrimSpace(src) + } + return + } + // If the error is that the source file didn't begin with a + // declaration, fall through to try as a statement list. + // Stop and return on any other error. + if !strings.Contains(err.Error(), "expected declaration") { + return + } + + // If this is a statement list, make it a source file + // by inserting a package clause and turning the list + // into a function body. This handles expressions too. + // Insert using a ;, not a newline, so that the line numbers + // in fsrc match the ones in src. Add an extra '\n' before the '}' + // to make sure comments are flushed before the '}'. + fsrc := append(append([]byte("package p; func _() {"), src...), '\n', '\n', '}') + file, err = parser.ParseFile(fset, filename, fsrc, parserMode) + if err == nil { + sourceAdj = func(src []byte, indent int) []byte { + // Cap adjusted indent to zero. + if indent < 0 { + indent = 0 + } + // Remove the wrapping. + // Gofmt has turned the ; into a \n\n. + // There will be two non-blank lines with indent, hence 2*indent. + src = src[2*indent+len("package p\n\nfunc _() {"):] + // Remove only the "}\n" suffix: remaining whitespaces will be trimmed anyway + src = src[:len(src)-len("}\n")] + return bytes.TrimSpace(src) + } + // Gofmt has also indented the function body one level. + // Adjust that with indentAdj. + indentAdj = -1 + } + + // Succeeded, or out of options. + return +} + +// format formats the given package file originally obtained from src +// and adjusts the result based on the original source via sourceAdj +// and indentAdj. +func format( + fset *token.FileSet, + file *ast.File, + sourceAdj func(src []byte, indent int) []byte, + indentAdj int, + src []byte, + cfg printer.Config, +) ([]byte, error) { + if sourceAdj == nil { + // Complete source file. + var buf bytes.Buffer + err := cfg.Fprint(&buf, fset, file) + if err != nil { + return nil, err + } + return buf.Bytes(), nil + } + + // Partial source file. + // Determine and prepend leading space. + i, j := 0, 0 + for j < len(src) && isSpace(src[j]) { + if src[j] == '\n' { + i = j + 1 // byte offset of last line in leading space + } + j++ + } + var res []byte + res = append(res, src[:i]...) + + // Determine and prepend indentation of first code line. + // Spaces are ignored unless there are no tabs, + // in which case spaces count as one tab. + indent := 0 + hasSpace := false + for _, b := range src[i:j] { + switch b { + case ' ': + hasSpace = true + case '\t': + indent++ + } + } + if indent == 0 && hasSpace { + indent = 1 + } + for i := 0; i < indent; i++ { + res = append(res, '\t') + } + + // Format the source. + // Write it without any leading and trailing space. + cfg.Indent = indent + indentAdj + var buf bytes.Buffer + err := cfg.Fprint(&buf, fset, file) + if err != nil { + return nil, err + } + out := sourceAdj(buf.Bytes(), cfg.Indent) + + // If the adjusted output is empty, the source + // was empty but (possibly) for white space. + // The result is the incoming source. + if len(out) == 0 { + return src, nil + } + + // Otherwise, append output to leading space. + res = append(res, out...) + + // Determine and append trailing space. + i = len(src) + for i > 0 && isSpace(src[i-1]) { + i-- + } + return append(res, src[i:]...), nil +} + +// isSpace reports whether the byte is a space character. +// isSpace defines a space as being among the following bytes: ' ', '\t', '\n' and '\r'. +func isSpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} diff --git a/libgo/go/cmd/gofmt/long_test.go b/libgo/go/cmd/gofmt/long_test.go index df9a878..237b860 100644 --- a/libgo/go/cmd/gofmt/long_test.go +++ b/libgo/go/cmd/gofmt/long_test.go @@ -15,7 +15,6 @@ import ( "go/ast" "go/printer" "go/token" - "internal/format" "io" "os" "path/filepath" @@ -33,7 +32,7 @@ var ( ) func gofmt(fset *token.FileSet, filename string, src *bytes.Buffer) error { - f, _, _, err := format.Parse(fset, filename, src.Bytes(), false) + f, _, _, err := parse(fset, filename, src.Bytes(), false) if err != nil { return err } @@ -61,7 +60,7 @@ func testFile(t *testing.T, b1, b2 *bytes.Buffer, filename string) { // exclude files w/ syntax errors (typically test cases) fset := token.NewFileSet() - if _, _, _, err = format.Parse(fset, filename, b1.Bytes(), false); err != nil { + if _, _, _, err = parse(fset, filename, b1.Bytes(), false); err != nil { if *verbose { fmt.Fprintf(os.Stderr, "ignoring %s\n", err) } diff --git a/libgo/go/cmd/gofmt/testdata/old.golden b/libgo/go/cmd/gofmt/testdata/old.golden deleted file mode 100644 index 95a0b72..0000000 --- a/libgo/go/cmd/gofmt/testdata/old.golden +++ /dev/null @@ -1,9 +0,0 @@ -package P - -func f() { - if x { - y - } else { - z - } -} diff --git a/libgo/go/cmd/gofmt/testdata/old.input b/libgo/go/cmd/gofmt/testdata/old.input deleted file mode 100644 index e24eed2..0000000 --- a/libgo/go/cmd/gofmt/testdata/old.input +++ /dev/null @@ -1,8 +0,0 @@ -package P - -func f() { - if x { - y - } else - z -} |