diff options
author | Ian Lance Taylor <iant@golang.org> | 2020-01-02 15:05:27 -0800 |
---|---|---|
committer | Ian Lance Taylor <iant@golang.org> | 2020-01-21 23:53:22 -0800 |
commit | 5a8ea165926cb0737ab03bc48c18dc5198ab5305 (patch) | |
tree | 962dc3357c57f019f85658f99e2e753e30201c27 /libgo/go/net | |
parent | 6ac6529e155c9baa0aaaed7aca06bd38ebda5b43 (diff) | |
download | gcc-5a8ea165926cb0737ab03bc48c18dc5198ab5305.zip gcc-5a8ea165926cb0737ab03bc48c18dc5198ab5305.tar.gz gcc-5a8ea165926cb0737ab03bc48c18dc5198ab5305.tar.bz2 |
libgo: update to Go1.14beta1
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/214297
Diffstat (limited to 'libgo/go/net')
118 files changed, 3879 insertions, 763 deletions
diff --git a/libgo/go/net/dial.go b/libgo/go/net/dial.go index 4d55a95..d8be1c2 100644 --- a/libgo/go/net/dial.go +++ b/libgo/go/net/dial.go @@ -23,6 +23,8 @@ const ( // The zero value for each field is equivalent to dialing // without that option. Dialing with the zero value of Dialer // is therefore equivalent to just calling the Dial function. +// +// It is safe to call Dialer's methods concurrently. type Dialer struct { // Timeout is the maximum amount of time a dial will wait for // a connect to complete. If Deadline is also set, it may fail @@ -527,20 +529,21 @@ func (sd *sysDialer) dialSerial(ctx context.Context, ras addrList) (Conn, error) default: } - deadline, _ := ctx.Deadline() - partialDeadline, err := partialDeadline(time.Now(), deadline, len(ras)-i) - if err != nil { - // Ran out of time. - if firstErr == nil { - firstErr = &OpError{Op: "dial", Net: sd.network, Source: sd.LocalAddr, Addr: ra, Err: err} - } - break - } dialCtx := ctx - if partialDeadline.Before(deadline) { - var cancel context.CancelFunc - dialCtx, cancel = context.WithDeadline(ctx, partialDeadline) - defer cancel() + if deadline, hasDeadline := ctx.Deadline(); hasDeadline { + partialDeadline, err := partialDeadline(time.Now(), deadline, len(ras)-i) + if err != nil { + // Ran out of time. + if firstErr == nil { + firstErr = &OpError{Op: "dial", Net: sd.network, Source: sd.LocalAddr, Addr: ra, Err: err} + } + break + } + if partialDeadline.Before(deadline) { + var cancel context.CancelFunc + dialCtx, cancel = context.WithDeadline(ctx, partialDeadline) + defer cancel() + } } c, err := sd.dialSingle(dialCtx, ra) diff --git a/libgo/go/net/dial_test.go b/libgo/go/net/dial_test.go index 1bf96fd..ae40079 100644 --- a/libgo/go/net/dial_test.go +++ b/libgo/go/net/dial_test.go @@ -14,6 +14,7 @@ import ( "io" "os" "runtime" + "strings" "sync" "testing" "time" @@ -154,7 +155,7 @@ func slowDialTCP(ctx context.Context, network string, laddr, raddr *TCPAddr) (*T return c, err } -func dialClosedPort() (actual, expected time.Duration) { +func dialClosedPort(t *testing.T) (actual, expected time.Duration) { // Estimate the expected time for this platform. // On Windows, dialing a closed port takes roughly 1 second, // but other platforms should be instantaneous. @@ -168,6 +169,7 @@ func dialClosedPort() (actual, expected time.Duration) { l, err := Listen("tcp", "127.0.0.1:0") if err != nil { + t.Logf("dialClosedPort: Listen failed: %v", err) return 999 * time.Hour, expected } addr := l.Addr().String() @@ -183,6 +185,7 @@ func dialClosedPort() (actual, expected time.Duration) { } elapsed := time.Now().Sub(startTime) if i == 2 { + t.Logf("dialClosedPort: measured delay %v", elapsed) return elapsed, expected } } @@ -195,7 +198,7 @@ func TestDialParallel(t *testing.T) { t.Skip("both IPv4 and IPv6 are required") } - closedPortDelay, expectClosedPortDelay := dialClosedPort() + closedPortDelay, expectClosedPortDelay := dialClosedPort(t) if closedPortDelay > expectClosedPortDelay { t.Errorf("got %v; want <= %v", closedPortDelay, expectClosedPortDelay) } @@ -316,8 +319,14 @@ func TestDialParallel(t *testing.T) { t.Errorf("#%d: got nil; want non-nil", i) } - expectElapsedMin := tt.expectElapsed - 95*time.Millisecond - expectElapsedMax := tt.expectElapsed + 95*time.Millisecond + // We used to always use 95 milliseconds as the slop, + // but that was flaky on Windows. See issue 35616. + slop := 95 * time.Millisecond + if fifth := tt.expectElapsed / 5; fifth > slop { + slop = fifth + } + expectElapsedMin := tt.expectElapsed - slop + expectElapsedMax := tt.expectElapsed + slop if elapsed < expectElapsedMin { t.Errorf("#%d: got %v; want >= %v", i, elapsed, expectElapsedMin) } else if elapsed > expectElapsedMax { @@ -639,9 +648,11 @@ func TestDialerLocalAddr(t *testing.T) { } c, err := d.Dial(tt.network, addr) if err == nil && tt.error != nil || err != nil && tt.error == nil { - // On Darwin this occasionally times out. - // We don't know why. Issue #22019. - if runtime.GOOS == "darwin" && tt.error == nil && os.IsTimeout(err) { + // A suspected kernel bug in macOS 10.12 occasionally results in + // timeout errors when dialing address ::1. The errors have not + // been observed on newer versions of the OS, so we don't plan to work + // around them. See https://golang.org/issue/22019. + if tt.raddr == "::1" && os.Getenv("GO_BUILDER_NAME") == "darwin-amd64-10_12" && os.IsTimeout(err) { t.Logf("ignoring timeout error on Darwin; see https://golang.org/issue/22019") } else { t.Errorf("%s %v->%s: got %v; want %v", tt.network, tt.laddr, tt.raddr, err, tt.error) @@ -664,7 +675,7 @@ func TestDialerDualStack(t *testing.T) { t.Skip("both IPv4 and IPv6 are required") } - closedPortDelay, expectClosedPortDelay := dialClosedPort() + closedPortDelay, expectClosedPortDelay := dialClosedPort(t) if closedPortDelay > expectClosedPortDelay { t.Errorf("got %v; want <= %v", closedPortDelay, expectClosedPortDelay) } @@ -757,17 +768,8 @@ func TestDialerKeepAlive(t *testing.T) { } func TestDialCancel(t *testing.T) { - switch testenv.Builder() { - case "linux-arm64-buildlet": - t.Skip("skipping on linux-arm64-buildlet; incompatible network config? issue 15191") - } mustHaveExternalNetwork(t) - if runtime.GOOS == "nacl" { - // nacl doesn't have external network access. - t.Skipf("skipping on %s", runtime.GOOS) - } - blackholeIPPort := JoinHostPort(slowDst4, "1234") if !supportsIPv4() { blackholeIPPort = JoinHostPort(slowDst6, "1234") @@ -810,6 +812,11 @@ func TestDialCancel(t *testing.T) { t.Error(perr) } if ticks < cancelTick { + // Using strings.Contains is ugly but + // may work on plan9 and windows. + if strings.Contains(err.Error(), "connection refused") { + t.Skipf("connection to %v failed fast with %v", blackholeIPPort, err) + } t.Fatalf("dial error after %d ticks (%d before cancel sent): %v", ticks, cancelTick-ticks, err) } @@ -923,7 +930,7 @@ func TestDialListenerAddr(t *testing.T) { func TestDialerControl(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } @@ -981,3 +988,32 @@ func mustHaveExternalNetwork(t *testing.T) { testenv.MustHaveExternalNetwork(t) } } + +type contextWithNonZeroDeadline struct { + context.Context +} + +func (contextWithNonZeroDeadline) Deadline() (time.Time, bool) { + // Return non-zero time.Time value with false indicating that no deadline is set. + return time.Unix(0, 0), false +} + +func TestDialWithNonZeroDeadline(t *testing.T) { + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + _, port, err := SplitHostPort(ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + + ctx := contextWithNonZeroDeadline{Context: context.Background()} + var dialer Dialer + c, err := dialer.DialContext(ctx, "tcp", JoinHostPort("", port)) + if err != nil { + t.Fatal(err) + } + c.Close() +} diff --git a/libgo/go/net/dnsclient_unix.go b/libgo/go/net/dnsclient_unix.go index b3284b8..da6baf3 100644 --- a/libgo/go/net/dnsclient_unix.go +++ b/libgo/go/net/dnsclient_unix.go @@ -765,6 +765,14 @@ func (r *Resolver) goLookupPTR(ctx context.Context, addr string) ([]string, erro } } if h.Type != dnsmessage.TypePTR { + err := p.SkipAnswer() + if err != nil { + return nil, &DNSError{ + Err: "cannot marshal DNS message", + Name: addr, + Server: server, + } + } continue } ptr, err := p.PTRResource() diff --git a/libgo/go/net/dnsclient_unix_test.go b/libgo/go/net/dnsclient_unix_test.go index b51d608..6d72817 100644 --- a/libgo/go/net/dnsclient_unix_test.go +++ b/libgo/go/net/dnsclient_unix_test.go @@ -1753,3 +1753,50 @@ func TestDNSUseTCP(t *testing.T) { t.Fatal("exchange failed:", err) } } + +// Issue 34660: PTR response with non-PTR answers should ignore non-PTR +func TestPTRandNonPTR(t *testing.T) { + fake := fakeDNSServer{ + rh: func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { + r := dnsmessage.Message{ + Header: dnsmessage.Header{ + ID: q.Header.ID, + Response: true, + RCode: dnsmessage.RCodeSuccess, + }, + Questions: q.Questions, + Answers: []dnsmessage.Resource{ + { + Header: dnsmessage.ResourceHeader{ + Name: q.Questions[0].Name, + Type: dnsmessage.TypePTR, + Class: dnsmessage.ClassINET, + }, + Body: &dnsmessage.PTRResource{ + PTR: dnsmessage.MustNewName("golang.org."), + }, + }, + { + Header: dnsmessage.ResourceHeader{ + Name: q.Questions[0].Name, + Type: dnsmessage.TypeTXT, + Class: dnsmessage.ClassINET, + }, + Body: &dnsmessage.TXTResource{ + TXT: []string{"PTR 8 6 60 ..."}, // fake RRSIG + }, + }, + }, + } + return r, nil + }, + } + r := Resolver{PreferGo: true, Dial: fake.DialContext} + names, err := r.lookupAddr(context.Background(), "192.0.2.123") + if err != nil { + t.Fatalf("LookupAddr: %v", err) + } + if want := []string{"golang.org."}; !reflect.DeepEqual(names, want) { + t.Errorf("names = %q; want %q", names, want) + } +} diff --git a/libgo/go/net/error_nacl.go b/libgo/go/net/error_nacl.go deleted file mode 100644 index caad133..0000000 --- a/libgo/go/net/error_nacl.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -func isConnError(err error) bool { - return false -} diff --git a/libgo/go/net/error_posix.go b/libgo/go/net/error_posix.go index 0ea26e9..c13d5bc 100644 --- a/libgo/go/net/error_posix.go +++ b/libgo/go/net/error_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris windows package net diff --git a/libgo/go/net/error_test.go b/libgo/go/net/error_test.go index c4fee5aa..89dcc2e 100644 --- a/libgo/go/net/error_test.go +++ b/libgo/go/net/error_test.go @@ -185,7 +185,7 @@ func TestDialError(t *testing.T) { func TestProtocolDialError(t *testing.T) { switch runtime.GOOS { - case "nacl", "solaris", "illumos": + case "solaris", "illumos": t.Skipf("not supported on %s", runtime.GOOS) } @@ -214,7 +214,7 @@ func TestProtocolDialError(t *testing.T) { func TestDialAddrError(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } if !supportsIPv4() || !supportsIPv6() { @@ -376,7 +376,7 @@ func TestListenPacketError(t *testing.T) { func TestProtocolListenError(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } diff --git a/libgo/go/net/example_test.go b/libgo/go/net/example_test.go new file mode 100644 index 0000000..ef8c38f --- /dev/null +++ b/libgo/go/net/example_test.go @@ -0,0 +1,160 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net_test + +import ( + "context" + "fmt" + "io" + "log" + "net" + "time" +) + +func ExampleListener() { + // Listen on TCP port 2000 on all available unicast and + // anycast IP addresses of the local system. + l, err := net.Listen("tcp", ":2000") + if err != nil { + log.Fatal(err) + } + defer l.Close() + for { + // Wait for a connection. + conn, err := l.Accept() + if err != nil { + log.Fatal(err) + } + // Handle the connection in a new goroutine. + // The loop then returns to accepting, so that + // multiple connections may be served concurrently. + go func(c net.Conn) { + // Echo all incoming data. + io.Copy(c, c) + // Shut down the connection. + c.Close() + }(conn) + } +} + +func ExampleDialer() { + var d net.Dialer + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + conn, err := d.DialContext(ctx, "tcp", "localhost:12345") + if err != nil { + log.Fatalf("Failed to dial: %v", err) + } + defer conn.Close() + + if _, err := conn.Write([]byte("Hello, World!")); err != nil { + log.Fatal(err) + } +} + +func ExampleIPv4() { + fmt.Println(net.IPv4(8, 8, 8, 8)) + + // Output: + // 8.8.8.8 +} + +func ExampleParseCIDR() { + ipv4Addr, ipv4Net, err := net.ParseCIDR("192.0.2.1/24") + if err != nil { + log.Fatal(err) + } + fmt.Println(ipv4Addr) + fmt.Println(ipv4Net) + + ipv6Addr, ipv6Net, err := net.ParseCIDR("2001:db8:a0b:12f0::1/32") + if err != nil { + log.Fatal(err) + } + fmt.Println(ipv6Addr) + fmt.Println(ipv6Net) + + // Output: + // 192.0.2.1 + // 192.0.2.0/24 + // 2001:db8:a0b:12f0::1 + // 2001:db8::/32 +} + +func ExampleParseIP() { + fmt.Println(net.ParseIP("192.0.2.1")) + fmt.Println(net.ParseIP("2001:db8::68")) + fmt.Println(net.ParseIP("192.0.2")) + + // Output: + // 192.0.2.1 + // 2001:db8::68 + // <nil> +} + +func ExampleIP_DefaultMask() { + ip := net.ParseIP("192.0.2.1") + fmt.Println(ip.DefaultMask()) + + // Output: + // ffffff00 +} + +func ExampleIP_Mask() { + ipv4Addr := net.ParseIP("192.0.2.1") + // This mask corresponds to a /24 subnet for IPv4. + ipv4Mask := net.CIDRMask(24, 32) + fmt.Println(ipv4Addr.Mask(ipv4Mask)) + + ipv6Addr := net.ParseIP("2001:db8:a0b:12f0::1") + // This mask corresponds to a /32 subnet for IPv6. + ipv6Mask := net.CIDRMask(32, 128) + fmt.Println(ipv6Addr.Mask(ipv6Mask)) + + // Output: + // 192.0.2.0 + // 2001:db8:: +} + +func ExampleCIDRMask() { + // This mask corresponds to a /31 subnet for IPv4. + fmt.Println(net.CIDRMask(31, 32)) + + // This mask corresponds to a /64 subnet for IPv6. + fmt.Println(net.CIDRMask(64, 128)) + + // Output: + // fffffffe + // ffffffffffffffff0000000000000000 +} + +func ExampleIPv4Mask() { + fmt.Println(net.IPv4Mask(255, 255, 255, 0)) + + // Output: + // ffffff00 +} + +func ExampleUDPConn_WriteTo() { + // Unlike Dial, ListenPacket creates a connection without any + // association with peers. + conn, err := net.ListenPacket("udp", ":0") + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + dst, err := net.ResolveUDPAddr("udp", "192.0.2.1:2000") + if err != nil { + log.Fatal(err) + } + + // The connection can write data to the desired address. + _, err = conn.WriteTo([]byte("data"), dst) + if err != nil { + log.Fatal(err) + } +} diff --git a/libgo/go/net/fd_unix.go b/libgo/go/net/fd_unix.go index 286d3f1..117f5a9 100644 --- a/libgo/go/net/fd_unix.go +++ b/libgo/go/net/fd_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd linux nacl netbsd openbsd solaris +// +build aix darwin dragonfly freebsd hurd linux netbsd openbsd solaris package net @@ -96,7 +96,7 @@ func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (rsa sysc if err := fd.pfd.Init(fd.net, true); err != nil { return nil, err } - if deadline, _ := ctx.Deadline(); !deadline.IsZero() { + if deadline, hasDeadline := ctx.Deadline(); hasDeadline { fd.pfd.SetWriteDeadline(deadline) defer fd.pfd.SetWriteDeadline(noDeadline) } @@ -248,7 +248,7 @@ func (fd *netFD) accept() (netfd *netFD, err error) { return nil, err } if err = netfd.init(); err != nil { - fd.Close() + netfd.Close() return nil, err } lsa, _ := syscall.Getsockname(netfd.pfd.Sysfd) diff --git a/libgo/go/net/file.go b/libgo/go/net/file.go index 81a44e1..c13332c 100644 --- a/libgo/go/net/file.go +++ b/libgo/go/net/file.go @@ -6,7 +6,7 @@ package net import "os" -// BUG(mikio): On JS, NaCl and Windows, the FileConn, FileListener and +// BUG(mikio): On JS and Windows, the FileConn, FileListener and // FilePacketConn functions are not implemented. type fileAddr string diff --git a/libgo/go/net/file_stub.go b/libgo/go/net/file_stub.go index 2256608..bfb8100 100644 --- a/libgo/go/net/file_stub.go +++ b/libgo/go/net/file_stub.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build nacl js,wasm +// +build js,wasm package net diff --git a/libgo/go/net/file_test.go b/libgo/go/net/file_test.go index cd71774..8c09c0d 100644 --- a/libgo/go/net/file_test.go +++ b/libgo/go/net/file_test.go @@ -31,7 +31,7 @@ var fileConnTests = []struct { func TestFileConn(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9", "windows": + case "plan9", "windows": t.Skipf("not supported on %s", runtime.GOOS) } @@ -138,7 +138,7 @@ var fileListenerTests = []struct { func TestFileListener(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9", "windows": + case "plan9", "windows": t.Skipf("not supported on %s", runtime.GOOS) } @@ -230,7 +230,7 @@ var filePacketConnTests = []struct { func TestFilePacketConn(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9", "windows": + case "plan9", "windows": t.Skipf("not supported on %s", runtime.GOOS) } @@ -297,7 +297,7 @@ func TestFilePacketConn(t *testing.T) { // Issue 24483. func TestFileCloseRace(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9", "windows": + case "plan9", "windows": t.Skipf("not supported on %s", runtime.GOOS) } if !testableNetwork("tcp") { diff --git a/libgo/go/net/hook_unix.go b/libgo/go/net/hook_unix.go index aaa6922..780b23c 100644 --- a/libgo/go/net/hook_unix.go +++ b/libgo/go/net/hook_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris package net diff --git a/libgo/go/net/http/npn_test.go b/libgo/go/net/http/alpn_test.go index 618bdbe..618bdbe 100644 --- a/libgo/go/net/http/npn_test.go +++ b/libgo/go/net/http/alpn_test.go diff --git a/libgo/go/net/http/cgi/host_test.go b/libgo/go/net/http/cgi/host_test.go index 25882de..9f1716b 100644 --- a/libgo/go/net/http/cgi/host_test.go +++ b/libgo/go/net/http/cgi/host_test.go @@ -456,6 +456,23 @@ func TestDirUnix(t *testing.T) { runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) } +func findPerl(t *testing.T) string { + t.Helper() + perl, err := exec.LookPath("perl") + if err != nil { + t.Skip("Skipping test: perl not found.") + } + perl, _ = filepath.Abs(perl) + + cmd := exec.Command(perl, "-e", "print 123") + cmd.Env = []string{"PATH=/garbage"} + out, err := cmd.Output() + if err != nil || string(out) != "123" { + t.Skipf("Skipping test: %s is not functional", perl) + } + return perl +} + func TestDirWindows(t *testing.T) { if runtime.GOOS != "windows" { t.Skip("Skipping windows specific test.") @@ -463,13 +480,7 @@ func TestDirWindows(t *testing.T) { cgifile, _ := filepath.Abs("testdata/test.cgi") - var perl string - var err error - perl, err = exec.LookPath("perl") - if err != nil { - t.Skip("Skipping test: perl not found.") - } - perl, _ = filepath.Abs(perl) + perl := findPerl(t) cwd, _ := os.Getwd() h := &Handler{ @@ -506,13 +517,7 @@ func TestEnvOverride(t *testing.T) { check(t) cgifile, _ := filepath.Abs("testdata/test.cgi") - var perl string - var err error - perl, err = exec.LookPath("perl") - if err != nil { - t.Skipf("Skipping test: perl not found.") - } - perl, _ = filepath.Abs(perl) + perl := findPerl(t) cwd, _ := os.Getwd() h := &Handler{ diff --git a/libgo/go/net/http/client.go b/libgo/go/net/http/client.go index 65a9d51..6a8c59a 100644 --- a/libgo/go/net/http/client.go +++ b/libgo/go/net/http/client.go @@ -10,6 +10,7 @@ package http import ( + "context" "crypto/tls" "encoding/base64" "errors" @@ -18,6 +19,7 @@ import ( "io/ioutil" "log" "net/url" + "reflect" "sort" "strings" "sync" @@ -238,7 +240,7 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, d username := u.Username() password, _ := u.Password() forkReq() - req.Header = ireq.Header.Clone() + req.Header = cloneOrMakeHeader(ireq.Header) req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) } @@ -273,46 +275,95 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, d return resp, nil, nil } -// setRequestCancel sets the Cancel field of req, if deadline is -// non-zero. The RoundTripper's type is used to determine whether the legacy -// CancelRequest behavior should be used. +// timeBeforeContextDeadline reports whether the non-zero Time t is +// before ctx's deadline, if any. If ctx does not have a deadline, it +// always reports true (the deadline is considered infinite). +func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool { + d, ok := ctx.Deadline() + if !ok { + return true + } + return t.Before(d) +} + +// knownRoundTripperImpl reports whether rt is a RoundTripper that's +// maintained by the Go team and known to implement the latest +// optional semantics (notably contexts). +func knownRoundTripperImpl(rt RoundTripper) bool { + switch rt.(type) { + case *Transport, *http2Transport: + return true + } + // There's a very minor chance of a false positive with this. + // Insted of detecting our golang.org/x/net/http2.Transport, + // it might detect a Transport type in a different http2 + // package. But I know of none, and the only problem would be + // some temporarily leaked goroutines if the transport didn't + // support contexts. So this is a good enough heuristic: + if reflect.TypeOf(rt).String() == "*http2.Transport" { + return true + } + return false +} + +// setRequestCancel sets req.Cancel and adds a deadline context to req +// if deadline is non-zero. The RoundTripper's type is used to +// determine whether the legacy CancelRequest behavior should be used. // // As background, there are three ways to cancel a request: // First was Transport.CancelRequest. (deprecated) -// Second was Request.Cancel (this mechanism). +// Second was Request.Cancel. // Third was Request.Context. +// This function populates the second and third, and uses the first if it really needs to. func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) { if deadline.IsZero() { return nop, alwaysFalse } + knownTransport := knownRoundTripperImpl(rt) + oldCtx := req.Context() + if req.Cancel == nil && knownTransport { + // If they already had a Request.Context that's + // expiring sooner, do nothing: + if !timeBeforeContextDeadline(deadline, oldCtx) { + return nop, alwaysFalse + } + + var cancelCtx func() + req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) + return cancelCtx, func() bool { return time.Now().After(deadline) } + } initialReqCancel := req.Cancel // the user's original Request.Cancel, if any + var cancelCtx func() + if oldCtx := req.Context(); timeBeforeContextDeadline(deadline, oldCtx) { + req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) + } + cancel := make(chan struct{}) req.Cancel = cancel doCancel := func() { - // The newer way (the second way in the func comment): + // The second way in the func comment above: close(cancel) - - // The legacy compatibility way, used only - // for RoundTripper implementations written - // before Go 1.5 or Go 1.6. - type canceler interface { - CancelRequest(*Request) - } - switch v := rt.(type) { - case *Transport, *http2Transport: - // Do nothing. The net/http package's transports - // support the new Request.Cancel channel - case canceler: + // The first way, used only for RoundTripper + // implementations written before Go 1.5 or Go 1.6. + type canceler interface{ CancelRequest(*Request) } + if v, ok := rt.(canceler); ok { v.CancelRequest(req) } } stopTimerCh := make(chan struct{}) var once sync.Once - stopTimer = func() { once.Do(func() { close(stopTimerCh) }) } + stopTimer = func() { + once.Do(func() { + close(stopTimerCh) + if cancelCtx != nil { + cancelCtx() + } + }) + } timer := time.NewTimer(time.Until(deadline)) var timedOut atomicBool @@ -383,8 +434,8 @@ func Get(url string) (resp *Response, err error) { // An error is returned if the Client's CheckRedirect function fails // or if there was an HTTP protocol error. A non-2xx response doesn't // cause an error. Any returned error will be of type *url.Error. The -// url.Error value's Timeout method will report true if request timed -// out or was canceled. +// url.Error value's Timeout method will report true if the request +// timed out. // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. @@ -668,7 +719,7 @@ func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { // The headers to copy are from the very initial request. // We use a closured callback to keep a reference to these original headers. var ( - ireqhdr = ireq.Header.Clone() + ireqhdr = cloneOrMakeHeader(ireq.Header) icookies map[string][]*Cookie ) if c.Jar != nil && ireq.Header.Get("Cookie") != "" { @@ -870,8 +921,7 @@ func (b *cancelTimerBody) Read(p []byte) (n int, err error) { } if b.reqDidTimeout() { err = &httpError{ - // TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancellation/ - err: err.Error() + " (Client.Timeout exceeded while reading body)", + err: err.Error() + " (Client.Timeout or context cancellation while reading body)", timeout: true, } } diff --git a/libgo/go/net/http/client_test.go b/libgo/go/net/http/client_test.go index de490bc..2b4f53f 100644 --- a/libgo/go/net/http/client_test.go +++ b/libgo/go/net/http/client_test.go @@ -221,27 +221,27 @@ func TestClientRedirects(t *testing.T) { c := ts.Client() _, err := c.Get(ts.URL) - if e, g := "Get /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { + if e, g := `Get "/?n=10": stopped after 10 redirects`, fmt.Sprintf("%v", err); e != g { t.Errorf("with default client Get, expected error %q, got %q", e, g) } // HEAD request should also have the ability to follow redirects. _, err = c.Head(ts.URL) - if e, g := "Head /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { + if e, g := `Head "/?n=10": stopped after 10 redirects`, fmt.Sprintf("%v", err); e != g { t.Errorf("with default client Head, expected error %q, got %q", e, g) } // Do should also follow redirects. greq, _ := NewRequest("GET", ts.URL, nil) _, err = c.Do(greq) - if e, g := "Get /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { + if e, g := `Get "/?n=10": stopped after 10 redirects`, fmt.Sprintf("%v", err); e != g { t.Errorf("with default client Do, expected error %q, got %q", e, g) } // Requests with an empty Method should also redirect (Issue 12705) greq.Method = "" _, err = c.Do(greq) - if e, g := "Get /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { + if e, g := `Get "/?n=10": stopped after 10 redirects`, fmt.Sprintf("%v", err); e != g { t.Errorf("with default client Do and empty Method, expected error %q, got %q", e, g) } @@ -1172,22 +1172,22 @@ func TestStripPasswordFromError(t *testing.T) { { desc: "Strip password from error message", in: "http://user:password@dummy.faketld/", - out: "Get http://user:***@dummy.faketld/: dummy impl", + out: `Get "http://user:***@dummy.faketld/": dummy impl`, }, { desc: "Don't Strip password from domain name", in: "http://user:password@password.faketld/", - out: "Get http://user:***@password.faketld/: dummy impl", + out: `Get "http://user:***@password.faketld/": dummy impl`, }, { desc: "Don't Strip password from path", in: "http://user:password@dummy.faketld/password", - out: "Get http://user:***@dummy.faketld/password: dummy impl", + out: `Get "http://user:***@dummy.faketld/password": dummy impl`, }, { desc: "Strip escaped password", in: "http://user:pa%2Fssword@dummy.faketld/", - out: "Get http://user:***@dummy.faketld/: dummy impl", + out: `Get "http://user:***@dummy.faketld/": dummy impl`, }, } for _, tC := range testCases { @@ -1274,7 +1274,7 @@ func testClientTimeout(t *testing.T, h2 bool) { } else if !ne.Timeout() { t.Errorf("net.Error.Timeout = false; want true") } - if got := ne.Error(); !strings.Contains(got, "Client.Timeout exceeded") { + if got := ne.Error(); !strings.Contains(got, "(Client.Timeout") { t.Errorf("error string = %q; missing timeout substring", got) } case <-time.After(failTime): @@ -1917,3 +1917,77 @@ func TestClientCloseIdleConnections(t *testing.T) { t.Error("not closed") } } + +func TestClientPropagatesTimeoutToContext(t *testing.T) { + errDial := errors.New("not actually dialing") + c := &Client{ + Timeout: 5 * time.Second, + Transport: &Transport{ + DialContext: func(ctx context.Context, netw, addr string) (net.Conn, error) { + deadline, ok := ctx.Deadline() + if !ok { + t.Error("no deadline") + } else { + t.Logf("deadline in %v", deadline.Sub(time.Now()).Round(time.Second/10)) + } + return nil, errDial + }, + }, + } + c.Get("https://example.tld/") +} + +func TestClientDoCanceledVsTimeout_h1(t *testing.T) { + testClientDoCanceledVsTimeout(t, h1Mode) +} + +func TestClientDoCanceledVsTimeout_h2(t *testing.T) { + testClientDoCanceledVsTimeout(t, h2Mode) +} + +// Issue 33545: lock-in the behavior promised by Client.Do's +// docs about request cancelation vs timing out. +func testClientDoCanceledVsTimeout(t *testing.T, h2 bool) { + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Write([]byte("Hello, World!")) + })) + defer cst.close() + + cases := []string{"timeout", "canceled"} + + for _, name := range cases { + t.Run(name, func(t *testing.T) { + var ctx context.Context + var cancel func() + if name == "timeout" { + ctx, cancel = context.WithTimeout(context.Background(), -time.Nanosecond) + } else { + ctx, cancel = context.WithCancel(context.Background()) + cancel() + } + defer cancel() + + req, _ := NewRequestWithContext(ctx, "GET", cst.ts.URL, nil) + _, err := cst.c.Do(req) + if err == nil { + t.Fatal("Unexpectedly got a nil error") + } + + ue := err.(*url.Error) + + var wantIsTimeout bool + var wantErr error = context.Canceled + if name == "timeout" { + wantErr = context.DeadlineExceeded + wantIsTimeout = true + } + if g, w := ue.Timeout(), wantIsTimeout; g != w { + t.Fatalf("url.Timeout() = %t, want %t", g, w) + } + if g, w := ue.Err, wantErr; g != w { + t.Errorf("url.Error.Err = %v; want %v", g, w) + } + }) + } +} diff --git a/libgo/go/net/http/clientserver_test.go b/libgo/go/net/http/clientserver_test.go index ee48fb0..70bcd0e 100644 --- a/libgo/go/net/http/clientserver_test.go +++ b/libgo/go/net/http/clientserver_test.go @@ -76,7 +76,16 @@ var optQuietLog = func(ts *httptest.Server) { ts.Config.ErrorLog = quietLog } +func optWithServerLog(lg *log.Logger) func(*httptest.Server) { + return func(ts *httptest.Server) { + ts.Config.ErrorLog = lg + } +} + func newClientServerTest(t *testing.T, h2 bool, h Handler, opts ...interface{}) *clientServerTest { + if h2 { + CondSkipHTTP2(t) + } cst := &clientServerTest{ t: t, h2: h2, diff --git a/libgo/go/net/http/clone.go b/libgo/go/net/http/clone.go index 5f2784d..3a3375b 100644 --- a/libgo/go/net/http/clone.go +++ b/libgo/go/net/http/clone.go @@ -62,3 +62,13 @@ func cloneMultipartFileHeader(fh *multipart.FileHeader) *multipart.FileHeader { fh2.Header = textproto.MIMEHeader(Header(fh.Header).Clone()) return fh2 } + +// cloneOrMakeHeader invokes Header.Clone but if the +// result is nil, it'll instead make and return a non-nil Header. +func cloneOrMakeHeader(hdr Header) Header { + clone := hdr.Clone() + if clone == nil { + clone = make(Header) + } + return clone +} diff --git a/libgo/go/net/http/cookie.go b/libgo/go/net/http/cookie.go index 91ff544..5c572d6d 100644 --- a/libgo/go/net/http/cookie.go +++ b/libgo/go/net/http/cookie.go @@ -353,6 +353,7 @@ func sanitizeCookieName(n string) string { return cookieNameSanitizer.Replace(n) } +// sanitizeCookieValue produces a suitable cookie-value from v. // https://tools.ietf.org/html/rfc6265#section-4.1.1 // cookie-value = *cookie-octet / ( DQUOTE *cookie-octet DQUOTE ) // cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / %x5D-7E @@ -360,8 +361,8 @@ func sanitizeCookieName(n string) string { // ; whitespace DQUOTE, comma, semicolon, // ; and backslash // We loosen this as spaces and commas are common in cookie values -// but we produce a quoted cookie-value in when value starts or ends -// with a comma or space. +// but we produce a quoted cookie-value if and only if v contains +// commas or spaces. // See https://golang.org/issue/7243 for the discussion. func sanitizeCookieValue(v string) string { v = sanitizeOrWarn("Cookie.Value", validCookieValueByte, v) diff --git a/libgo/go/net/http/export_test.go b/libgo/go/net/http/export_test.go index d265cd3..657ff9d 100644 --- a/libgo/go/net/http/export_test.go +++ b/libgo/go/net/http/export_test.go @@ -60,6 +60,12 @@ func init() { } } +func CondSkipHTTP2(t *testing.T) { + if omitBundledHTTP2 { + t.Skip("skipping HTTP/2 test when nethttpomithttp2 build tag in use") + } +} + var ( SetEnterRoundTripHook = hookSetter(&testHookEnterRoundTrip) SetRoundTripRetried = hookSetter(&testHookRoundTripRetried) @@ -208,6 +214,30 @@ func (t *Transport) PutIdleTestConn(scheme, addr string) bool { }) == nil } +// PutIdleTestConnH2 reports whether it was able to insert a fresh +// HTTP/2 persistConn for scheme, addr into the idle connection pool. +func (t *Transport) PutIdleTestConnH2(scheme, addr string, alt RoundTripper) bool { + key := connectMethodKey{"", scheme, addr, false} + + if t.MaxConnsPerHost > 0 { + // Transport is tracking conns-per-host. + // Increment connection count to account + // for new persistConn created below. + t.connsPerHostMu.Lock() + if t.connsPerHost == nil { + t.connsPerHost = make(map[connectMethodKey]int) + } + t.connsPerHost[key]++ + t.connsPerHostMu.Unlock() + } + + return t.tryPutIdleConn(&persistConn{ + t: t, + alt: alt, + cacheKey: key, + }) == nil +} + // All test hooks must be non-nil so they can be called directly, // but the tests use nil to mean hook disabled. func unnilTestHook(f *func()) { diff --git a/libgo/go/net/http/fs.go b/libgo/go/net/http/fs.go index 41d46dc..d214485 100644 --- a/libgo/go/net/http/fs.go +++ b/libgo/go/net/http/fs.go @@ -384,15 +384,18 @@ func checkIfUnmodifiedSince(r *Request, modtime time.Time) condResult { if ius == "" || isZeroTime(modtime) { return condNone } - if t, err := ParseTime(ius); err == nil { - // The Date-Modified header truncates sub-second precision, so - // use mtime < t+1s instead of mtime <= t to check for unmodified. - if modtime.Before(t.Add(1 * time.Second)) { - return condTrue - } - return condFalse + t, err := ParseTime(ius) + if err != nil { + return condNone } - return condNone + + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if modtime.Before(t) || modtime.Equal(t) { + return condTrue + } + return condFalse } func checkIfNoneMatch(w ResponseWriter, r *Request) condResult { @@ -436,9 +439,10 @@ func checkIfModifiedSince(r *Request, modtime time.Time) condResult { if err != nil { return condNone } - // The Date-Modified header truncates sub-second precision, so - // use mtime < t+1s instead of mtime <= t to check for unmodified. - if modtime.Before(t.Add(1 * time.Second)) { + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if modtime.Before(t) || modtime.Equal(t) { return condFalse } return condTrue @@ -582,17 +586,15 @@ func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirec } } - // redirect if the directory name doesn't end in a slash if d.IsDir() { url := r.URL.Path - if url[len(url)-1] != '/' { + // redirect if the directory name doesn't end in a slash + if url == "" || url[len(url)-1] != '/' { localRedirect(w, r, path.Base(url)+"/") return } - } - // use contents of index.html for directory, if present - if d.IsDir() { + // use contents of index.html for directory, if present index := strings.TrimSuffix(name, "/") + indexPage ff, err := fs.Open(index) if err == nil { @@ -612,7 +614,7 @@ func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirec writeNotModified(w) return } - w.Header().Set("Last-Modified", d.ModTime().UTC().Format(TimeFormat)) + setLastModified(w, d.ModTime()) dirList(w, r, f) return } diff --git a/libgo/go/net/http/fs_test.go b/libgo/go/net/http/fs_test.go index 82e13a4..435e34b 100644 --- a/libgo/go/net/http/fs_test.go +++ b/libgo/go/net/http/fs_test.go @@ -207,6 +207,18 @@ func TestServeFile_DotDot(t *testing.T) { } } +// Tests that this doesn't panic. (Issue 30165) +func TestServeFileDirPanicEmptyPath(t *testing.T) { + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/", nil) + req.URL.Path = "" + ServeFile(rec, req, "testdata") + res := rec.Result() + if res.StatusCode != 301 { + t.Errorf("code = %v; want 301", res.Status) + } +} + var fsRedirectTestData = []struct { original, redirect string }{ @@ -1110,21 +1122,13 @@ func TestLinuxSendfile(t *testing.T) { } defer ln.Close() - syscalls := "sendfile,sendfile64" - switch runtime.GOARCH { - case "mips64", "mips64le", "s390x", "alpha": - // strace on the above platforms doesn't support sendfile64 - // and will error out if we specify that with `-e trace='. - syscalls = "sendfile" - } - // Attempt to run strace, and skip on failure - this test requires SYS_PTRACE. - if err := exec.Command("strace", "-f", "-q", "-e", "trace="+syscalls, os.Args[0], "-test.run=^$").Run(); err != nil { + if err := exec.Command("strace", "-f", "-q", os.Args[0], "-test.run=^$").Run(); err != nil { t.Skipf("skipping; failed to run strace: %v", err) } var buf bytes.Buffer - child := exec.Command("strace", "-f", "-q", "-e", "trace="+syscalls, os.Args[0], "-test.run=TestLinuxSendfileChild") + child := exec.Command("strace", "-f", "-q", os.Args[0], "-test.run=TestLinuxSendfileChild") child.ExtraFiles = append(child.ExtraFiles, lnf) child.Env = append([]string{"GO_WANT_HELPER_PROCESS=1"}, os.Environ()...) child.Stdout = &buf @@ -1147,7 +1151,7 @@ func TestLinuxSendfile(t *testing.T) { Post(fmt.Sprintf("http://%s/quit", ln.Addr()), "", nil) child.Wait() - rx := regexp.MustCompile(`sendfile(64)?\(`) + rx := regexp.MustCompile(`\b(n64:)?sendfile(64)?\(`) out := buf.String() if !rx.MatchString(out) { t.Errorf("no sendfile system call found in:\n%s", out) diff --git a/libgo/go/net/http/h2_bundle.go b/libgo/go/net/http/h2_bundle.go index 21921ab..f03dbba 100644 --- a/libgo/go/net/http/h2_bundle.go +++ b/libgo/go/net/http/h2_bundle.go @@ -1,5 +1,7 @@ +// +build !nethttpomithttp2 + // Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. -//go:generate bundle -o h2_bundle.go -prefix http2 golang.org/x/net/http2 +// $ bundle -o=h2_bundle.go -prefix=http2 -tags=!nethttpomithttp2 golang.org/x/net/http2 // Package http2 implements the HTTP/2 protocol. // @@ -3465,6 +3467,7 @@ type http2pipe struct { mu sync.Mutex c sync.Cond // c.L lazily initialized to &p.mu b http2pipeBuffer // nil when done reading + unread int // bytes unread when done err error // read error once empty. non-nil means closed. breakErr error // immediate read error (caller doesn't see rest of b) donec chan struct{} // closed on error @@ -3481,7 +3484,7 @@ func (p *http2pipe) Len() int { p.mu.Lock() defer p.mu.Unlock() if p.b == nil { - return 0 + return p.unread } return p.b.Len() } @@ -3528,6 +3531,7 @@ func (p *http2pipe) Write(d []byte) (n int, err error) { return 0, http2errClosedPipeWrite } if p.breakErr != nil { + p.unread += len(d) return len(d), nil // discard when there is no reader } return p.b.Write(d) @@ -3565,6 +3569,9 @@ func (p *http2pipe) closeWithError(dst *error, err error, fn func()) { } p.readFn = fn if dst == &p.breakErr { + if p.b != nil { + p.unread += p.b.Len() + } p.b = nil } *dst = err @@ -3811,7 +3818,7 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { } } if !haveRequired { - return fmt.Errorf("http2: TLSConfig.CipherSuites is missing an HTTP/2-required AES_128_GCM_SHA256 cipher.") + return fmt.Errorf("http2: TLSConfig.CipherSuites is missing an HTTP/2-required AES_128_GCM_SHA256 cipher (need at least one of TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 or TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256).") } } @@ -3881,7 +3888,7 @@ type http2ServeConnOpts struct { } func (o *http2ServeConnOpts) context() context.Context { - if o.Context != nil { + if o != nil && o.Context != nil { return o.Context } return context.Background() @@ -5979,7 +5986,11 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { clen = strconv.Itoa(len(p)) } _, hasContentType := rws.snapHeader["Content-Type"] - if !hasContentType && http2bodyAllowedForStatus(rws.status) && len(p) > 0 { + // If the Content-Encoding is non-blank, we shouldn't + // sniff the body. See Issue golang.org/issue/31753. + ce := rws.snapHeader.Get("Content-Encoding") + hasCE := len(ce) > 0 + if !hasCE && !hasContentType && http2bodyAllowedForStatus(rws.status) && len(p) > 0 { ctype = DetectContentType(p) } var date string @@ -6088,7 +6099,7 @@ const http2TrailerPrefix = "Trailer:" // trailers. That worked for a while, until we found the first major // user of Trailers in the wild: gRPC (using them only over http2), // and gRPC libraries permit setting trailers mid-stream without -// predeclarnig them. So: change of plans. We still permit the old +// predeclaring them. So: change of plans. We still permit the old // way, but we also permit this hack: if a Header() key begins with // "Trailer:", the suffix of that key is a Trailer. Because ':' is an // invalid token byte anyway, there is no ambiguity. (And it's already @@ -6388,7 +6399,7 @@ func (sc *http2serverConn) startPush(msg *http2startPushRequest) { // PUSH_PROMISE frames MUST only be sent on a peer-initiated stream that // is in either the "open" or "half-closed (remote)" state. if msg.parent.state != http2stateOpen && msg.parent.state != http2stateHalfClosedRemote { - // responseWriter.Push checks that the stream is peer-initiaed. + // responseWriter.Push checks that the stream is peer-initiated. msg.done <- http2errStreamClosed return } @@ -6715,6 +6726,7 @@ type http2ClientConn struct { br *bufio.Reader fr *http2Framer lastActive time.Time + lastIdle time.Time // time last idle // Settings from peer: (also guarded by mu) maxFrameSize uint32 maxConcurrentStreams uint32 @@ -7092,7 +7104,7 @@ func (t *http2Transport) expectContinueTimeout() time.Duration { } func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { - return t.newClientConn(c, false) + return t.newClientConn(c, t.disableKeepAlives()) } func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2ClientConn, error) { @@ -7225,7 +7237,8 @@ func (cc *http2ClientConn) idleStateLocked() (st http2clientConnIdleState) { } st.canTakeNewRequest = cc.goAway == nil && !cc.closed && !cc.closing && maxConcurrentOkay && - int64(cc.nextStreamID)+2*int64(cc.pendingRequests) < math.MaxInt32 + int64(cc.nextStreamID)+2*int64(cc.pendingRequests) < math.MaxInt32 && + !cc.tooIdleLocked() st.freshConn = cc.nextStreamID == 1 && st.canTakeNewRequest return } @@ -7235,6 +7248,16 @@ func (cc *http2ClientConn) canTakeNewRequestLocked() bool { return st.canTakeNewRequest } +// tooIdleLocked reports whether this connection has been been sitting idle +// for too much wall time. +func (cc *http2ClientConn) tooIdleLocked() bool { + // The Round(0) strips the monontonic clock reading so the + // times are compared based on their wall time. We don't want + // to reuse a connection that's been sitting idle during + // VM/laptop suspend if monotonic time was also frozen. + return cc.idleTimeout != 0 && !cc.lastIdle.IsZero() && time.Since(cc.lastIdle.Round(0)) > cc.idleTimeout +} + // onIdleTimeout is called from a time.AfterFunc goroutine. It will // only be called when we're idle, but because we're coming from a new // goroutine, there could be a new request coming in at the same time, @@ -7654,6 +7677,7 @@ func (cc *http2ClientConn) awaitOpenSlotForRequest(req *Request) error { } return http2errClientConnUnusable } + cc.lastIdle = time.Time{} if int64(len(cc.streams))+1 <= int64(cc.maxConcurrentStreams) { if waitingForConn != nil { close(waitingForConn) @@ -7720,6 +7744,8 @@ var ( // abort request body write, but send stream reset of cancel. http2errStopReqBodyWriteAndCancel = errors.New("http2: canceling request") + + http2errReqBodyTooLong = errors.New("http2: request body larger than specified content length") ) func (cs *http2clientStream) writeRequestBody(body io.Reader, bodyCloser io.Closer) (err error) { @@ -7742,10 +7768,32 @@ func (cs *http2clientStream) writeRequestBody(body io.Reader, bodyCloser io.Clos req := cs.req hasTrailers := req.Trailer != nil + remainLen := http2actualContentLength(req) + hasContentLen := remainLen != -1 var sawEOF bool for !sawEOF { - n, err := body.Read(buf) + n, err := body.Read(buf[:len(buf)-1]) + if hasContentLen { + remainLen -= int64(n) + if remainLen == 0 && err == nil { + // The request body's Content-Length was predeclared and + // we just finished reading it all, but the underlying io.Reader + // returned the final chunk with a nil error (which is one of + // the two valid things a Reader can do at EOF). Because we'd prefer + // to send the END_STREAM bit early, double-check that we're actually + // at EOF. Subsequent reads should return (0, EOF) at this point. + // If either value is different, we return an error in one of two ways below. + var n1 int + n1, err = body.Read(buf[n:]) + remainLen -= int64(n1) + } + if remainLen < 0 { + err = http2errReqBodyTooLong + cc.writeStreamReset(cs.ID, http2ErrCodeCancel, err) + return err + } + } if err == io.EOF { sawEOF = true err = nil @@ -7958,7 +8006,29 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail if vv[0] == "" { continue } - + } else if strings.EqualFold(k, "cookie") { + // Per 8.1.2.5 To allow for better compression efficiency, the + // Cookie header field MAY be split into separate header fields, + // each with one or more cookie-pairs. + for _, v := range vv { + for { + p := strings.IndexByte(v, ';') + if p < 0 { + break + } + f("cookie", v[:p]) + p++ + // strip space after semicolon if any. + for p+1 <= len(v) && v[p] == ' ' { + p++ + } + v = v[p:] + } + if len(v) > 0 { + f("cookie", v) + } + } + continue } for _, v := range vv { @@ -8096,6 +8166,7 @@ func (cc *http2ClientConn) streamByID(id uint32, andRemove bool) *http2clientStr delete(cc.streams, id) if len(cc.streams) == 0 && cc.idleTimer != nil { cc.idleTimer.Reset(cc.idleTimeout) + cc.lastIdle = time.Now() } close(cs.done) // Wake up checkResetOrDone via clientStream.awaitFlowControl and @@ -9846,7 +9917,7 @@ func (n *http2priorityNode) addBytes(b int64) { } // walkReadyInOrder iterates over the tree in priority order, calling f for each node -// with a non-empty write queue. When f returns true, this funcion returns true and the +// with a non-empty write queue. When f returns true, this function returns true and the // walk halts. tmp is used as scratch space for sorting. // // f(n, openParent) takes two arguments: the node to visit, n, and a bool that is true @@ -10163,7 +10234,8 @@ type http2randomWriteScheduler struct { zero http2writeQueue // sq contains the stream-specific queues, keyed by stream ID. - // When a stream is idle or closed, it's deleted from the map. + // When a stream is idle, closed, or emptied, it's deleted + // from the map. sq map[uint32]*http2writeQueue // pool of empty queues for reuse. @@ -10207,8 +10279,12 @@ func (ws *http2randomWriteScheduler) Pop() (http2FrameWriteRequest, bool) { return ws.zero.shift(), true } // Iterate over all non-idle streams until finding one that can be consumed. - for _, q := range ws.sq { + for streamID, q := range ws.sq { if wr, ok := q.consume(math.MaxInt32); ok { + if q.empty() { + delete(ws.sq, streamID) + ws.queuePool.put(q) + } return wr, true } } diff --git a/libgo/go/net/http/header.go b/libgo/go/net/http/header.go index 230ca03..b9b5391 100644 --- a/libgo/go/net/http/header.go +++ b/libgo/go/net/http/header.go @@ -40,13 +40,21 @@ func (h Header) Set(key, value string) { // Get gets the first value associated with the given key. If // there are no values associated with the key, Get returns "". // It is case insensitive; textproto.CanonicalMIMEHeaderKey is -// used to canonicalize the provided key. To access multiple -// values of a key, or to use non-canonical keys, access the -// map directly. +// used to canonicalize the provided key. To use non-canonical keys, +// access the map directly. func (h Header) Get(key string) string { return textproto.MIMEHeader(h).Get(key) } +// Values returns all values associated with the given key. +// It is case insensitive; textproto.CanonicalMIMEHeaderKey is +// used to canonicalize the provided key. To use non-canonical +// keys, access the map directly. +// The returned slice is not a copy. +func (h Header) Values(key string) []string { + return textproto.MIMEHeader(h).Values(key) +} + // get is like Get, but key must already be in CanonicalHeaderKey form. func (h Header) get(key string) string { if v := h[key]; len(v) > 0 { @@ -170,6 +178,7 @@ func (h Header) sortedKeyValues(exclude map[string]bool) (kvs []keyValues, hs *h // WriteSubset writes a header in wire format. // If exclude is not nil, keys where exclude[key] == true are not written. +// Keys are not canonicalized before checking the exclude map. func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) error { return h.writeSubset(w, exclude, nil) } diff --git a/libgo/go/net/http/header_test.go b/libgo/go/net/http/header_test.go index a82504a..ad8ab9b 100644 --- a/libgo/go/net/http/header_test.go +++ b/libgo/go/net/http/header_test.go @@ -7,6 +7,7 @@ package http import ( "bytes" "internal/race" + "reflect" "runtime" "testing" "time" @@ -220,3 +221,34 @@ func TestHeaderWriteSubsetAllocs(t *testing.T) { t.Errorf("allocs = %g; want 0", n) } } + +// Issue 34878: test that every call to +// cloneOrMakeHeader never returns a nil Header. +func TestCloneOrMakeHeader(t *testing.T) { + tests := []struct { + name string + in, want Header + }{ + {"nil", nil, Header{}}, + {"empty", Header{}, Header{}}, + { + name: "non-empty", + in: Header{"foo": {"bar"}}, + want: Header{"foo": {"bar"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := cloneOrMakeHeader(tt.in) + if got == nil { + t.Fatal("unexpected nil Header") + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("Got: %#v\nWant: %#v", got, tt.want) + } + got.Add("A", "B") + got.Get("A") + }) + } +} diff --git a/libgo/go/net/http/http.go b/libgo/go/net/http/http.go index 3510fe6..89e86d8 100644 --- a/libgo/go/net/http/http.go +++ b/libgo/go/net/http/http.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:generate bundle -o=h2_bundle.go -prefix=http2 -tags=!nethttpomithttp2 golang.org/x/net/http2 + package http import ( @@ -22,6 +24,11 @@ const maxInt64 = 1<<63 - 1 // immediate cancellation of network operations. var aLongTimeAgo = time.Unix(1, 0) +// omitBundledHTTP2 is set by omithttp2.go when the nethttpomithttp2 +// build tag is set. That means h2_bundle.go isn't compiled in and we +// shouldn't try to use it. +var omitBundledHTTP2 bool + // TODO(bradfitz): move common stuff here. The other files have accumulated // generic http stuff in random places. diff --git a/libgo/go/net/http/http_test.go b/libgo/go/net/http/http_test.go index 8f466bb..f4ea52d 100644 --- a/libgo/go/net/http/http_test.go +++ b/libgo/go/net/http/http_test.go @@ -9,6 +9,7 @@ package http import ( "bytes" "internal/testenv" + "net/url" "os/exec" "reflect" "testing" @@ -109,3 +110,54 @@ func TestCmdGoNoHTTPServer(t *testing.T) { } } } + +// Tests that the nethttpomithttp2 build tag doesn't rot too much, +// even if there's not a regular builder on it. +func TestOmitHTTP2(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + t.Parallel() + goTool := testenv.GoToolPath(t) + out, err := exec.Command(goTool, "test", "-short", "-tags=nethttpomithttp2", "net/http").CombinedOutput() + if err != nil { + t.Fatalf("go test -short failed: %v, %s", err, out) + } +} + +// Tests that the nethttpomithttp2 build tag at least type checks +// in short mode. +// The TestOmitHTTP2 test above actually runs tests (in long mode). +func TestOmitHTTP2Vet(t *testing.T) { + t.Parallel() + goTool := testenv.GoToolPath(t) + out, err := exec.Command(goTool, "vet", "-tags=nethttpomithttp2", "net/http").CombinedOutput() + if err != nil { + t.Fatalf("go vet failed: %v, %s", err, out) + } +} + +var valuesCount int + +func BenchmarkCopyValues(b *testing.B) { + b.ReportAllocs() + src := url.Values{ + "a": {"1", "2", "3", "4", "5"}, + "b": {"2", "2", "3", "4", "5"}, + "c": {"3", "2", "3", "4", "5"}, + "d": {"4", "2", "3", "4", "5"}, + "e": {"1", "1", "2", "3", "4", "5", "6", "7", "abcdef", "l", "a", "b", "c", "d", "z"}, + "j": {"1", "2"}, + "m": nil, + } + for i := 0; i < b.N; i++ { + dst := url.Values{"a": {"b"}, "b": {"2"}, "c": {"3"}, "d": {"4"}, "j": nil, "m": {"x"}} + copyValues(dst, src) + if valuesCount = len(dst["a"]); valuesCount != 6 { + b.Fatalf(`%d items in dst["a"] but expected 6`, valuesCount) + } + } + if valuesCount == 0 { + b.Fatal("Benchmark wasn't run") + } +} diff --git a/libgo/go/net/http/httptest/example_test.go b/libgo/go/net/http/httptest/example_test.go new file mode 100644 index 0000000..54e77db --- /dev/null +++ b/libgo/go/net/http/httptest/example_test.go @@ -0,0 +1,100 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httptest_test + +import ( + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" +) + +func ExampleResponseRecorder() { + handler := func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "<html><body>Hello World!</body></html>") + } + + req := httptest.NewRequest("GET", "http://example.com/foo", nil) + w := httptest.NewRecorder() + handler(w, req) + + resp := w.Result() + body, _ := ioutil.ReadAll(resp.Body) + + fmt.Println(resp.StatusCode) + fmt.Println(resp.Header.Get("Content-Type")) + fmt.Println(string(body)) + + // Output: + // 200 + // text/html; charset=utf-8 + // <html><body>Hello World!</body></html> +} + +func ExampleServer() { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, client") + })) + defer ts.Close() + + res, err := http.Get(ts.URL) + if err != nil { + log.Fatal(err) + } + greeting, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s", greeting) + // Output: Hello, client +} + +func ExampleServer_hTTP2() { + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello, %s", r.Proto) + })) + ts.EnableHTTP2 = true + ts.StartTLS() + defer ts.Close() + + res, err := ts.Client().Get(ts.URL) + if err != nil { + log.Fatal(err) + } + greeting, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s", greeting) + + // Output: Hello, HTTP/2.0 +} + +func ExampleNewTLSServer() { + ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, client") + })) + defer ts.Close() + + client := ts.Client() + res, err := client.Get(ts.URL) + if err != nil { + log.Fatal(err) + } + + greeting, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s", greeting) + // Output: Hello, client +} diff --git a/libgo/go/net/http/httptest/server.go b/libgo/go/net/http/httptest/server.go index 485a4a5..8ebf681 100644 --- a/libgo/go/net/http/httptest/server.go +++ b/libgo/go/net/http/httptest/server.go @@ -27,6 +27,11 @@ type Server struct { URL string // base URL of form http://ipaddr:port with no trailing slash Listener net.Listener + // EnableHTTP2 controls whether HTTP/2 is enabled + // on the server. It must be set between calling + // NewUnstartedServer and calling Server.StartTLS. + EnableHTTP2 bool + // TLS is the optional TLS configuration, populated with a new config // after TLS is started. If set on an unstarted server before StartTLS // is called, existing fields are copied into the new config. @@ -151,7 +156,11 @@ func (s *Server) StartTLS() { s.TLS = new(tls.Config) } if s.TLS.NextProtos == nil { - s.TLS.NextProtos = []string{"http/1.1"} + nextProtos := []string{"http/1.1"} + if s.EnableHTTP2 { + nextProtos = []string{"h2"} + } + s.TLS.NextProtos = nextProtos } if len(s.TLS.Certificates) == 0 { s.TLS.Certificates = []tls.Certificate{cert} @@ -166,6 +175,7 @@ func (s *Server) StartTLS() { TLSClientConfig: &tls.Config{ RootCAs: certpool, }, + ForceAttemptHTTP2: s.EnableHTTP2, } s.Listener = tls.NewListener(s.Listener, s.TLS) s.URL = "https://" + s.Listener.Addr().String() diff --git a/libgo/go/net/http/httptest/server_test.go b/libgo/go/net/http/httptest/server_test.go index 8ab50cd..0aad15c 100644 --- a/libgo/go/net/http/httptest/server_test.go +++ b/libgo/go/net/http/httptest/server_test.go @@ -202,3 +202,39 @@ func TestServerZeroValueClose(t *testing.T) { ts.Close() // tests that it doesn't panic } + +func TestTLSServerWithHTTP2(t *testing.T) { + modes := []struct { + name string + wantProto string + }{ + {"http1", "HTTP/1.1"}, + {"http2", "HTTP/2.0"}, + } + + for _, tt := range modes { + t.Run(tt.name, func(t *testing.T) { + cst := NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Proto", r.Proto) + })) + + switch tt.name { + case "http2": + cst.EnableHTTP2 = true + cst.StartTLS() + default: + cst.Start() + } + + defer cst.Close() + + res, err := cst.Client().Get(cst.URL) + if err != nil { + t.Fatalf("Failed to make request: %v", err) + } + if g, w := res.Header.Get("X-Proto"), tt.wantProto; g != w { + t.Fatalf("X-Proto header mismatch:\n\tgot: %q\n\twant: %q", g, w) + } + }) + } +} diff --git a/libgo/go/net/http/httptrace/trace.go b/libgo/go/net/http/httptrace/trace.go index 8b377ed..6a5cbac 100644 --- a/libgo/go/net/http/httptrace/trace.go +++ b/libgo/go/net/http/httptrace/trace.go @@ -133,8 +133,8 @@ type ClientTrace struct { ConnectDone func(network, addr string, err error) // TLSHandshakeStart is called when the TLS handshake is started. When - // connecting to a HTTPS site via a HTTP proxy, the handshake happens after - // the CONNECT request is processed by the proxy. + // connecting to an HTTPS site via an HTTP proxy, the handshake happens + // after the CONNECT request is processed by the proxy. TLSHandshakeStart func() // TLSHandshakeDone is called after the TLS handshake with either the diff --git a/libgo/go/net/http/httputil/dump.go b/libgo/go/net/http/httputil/dump.go index 7104c37..c97be06 100644 --- a/libgo/go/net/http/httputil/dump.go +++ b/libgo/go/net/http/httputil/dump.go @@ -24,7 +24,7 @@ import ( // It returns an error if the initial slurp of all bytes fails. It does not attempt // to make the returned ReadClosers have identical error-matching behavior. func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) { - if b == http.NoBody { + if b == nil || b == http.NoBody { // No copying needed. Preserve the magic sentinel meaning of NoBody. return http.NoBody, http.NoBody, nil } @@ -60,16 +60,28 @@ func (b neverEnding) Read(p []byte) (n int, err error) { return len(p), nil } +// outGoingLength is a copy of the unexported +// (*http.Request).outgoingLength method. +func outgoingLength(req *http.Request) int64 { + if req.Body == nil || req.Body == http.NoBody { + return 0 + } + if req.ContentLength != 0 { + return req.ContentLength + } + return -1 +} + // DumpRequestOut is like DumpRequest but for outgoing client requests. It // includes any headers that the standard http.Transport adds, such as // User-Agent. func DumpRequestOut(req *http.Request, body bool) ([]byte, error) { save := req.Body dummyBody := false - if !body || req.Body == nil { - req.Body = nil - if req.ContentLength != 0 { - req.Body = ioutil.NopCloser(io.LimitReader(neverEnding('x'), req.ContentLength)) + if !body { + contentLength := outgoingLength(req) + if contentLength != 0 { + req.Body = ioutil.NopCloser(io.LimitReader(neverEnding('x'), contentLength)) dummyBody = true } } else { @@ -111,6 +123,10 @@ func DumpRequestOut(req *http.Request, body bool) ([]byte, error) { } defer t.CloseIdleConnections() + // We need this channel to ensure that the reader + // goroutine exits if t.RoundTrip returns an error. + // See golang.org/issue/32571. + quitReadCh := make(chan struct{}) // Wait for the request before replying with a dummy response: go func() { req, err := http.ReadRequest(bufio.NewReader(pr)) @@ -120,13 +136,18 @@ func DumpRequestOut(req *http.Request, body bool) ([]byte, error) { io.Copy(ioutil.Discard, req.Body) req.Body.Close() } - dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n") + select { + case dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n"): + case <-quitReadCh: + } }() _, err := t.RoundTrip(reqSend) req.Body = save if err != nil { + pw.Close() + quitReadCh <- struct{}{} return nil, err } dump := buf.Bytes() diff --git a/libgo/go/net/http/httputil/dump_test.go b/libgo/go/net/http/httputil/dump_test.go index 97954ca..ead56bc 100644 --- a/libgo/go/net/http/httputil/dump_test.go +++ b/libgo/go/net/http/httputil/dump_test.go @@ -17,6 +17,12 @@ import ( "testing" ) +type eofReader struct{} + +func (n eofReader) Close() error { return nil } + +func (n eofReader) Read([]byte) (int, error) { return 0, io.EOF } + type dumpTest struct { // Either Req or GetReq can be set/nil but not both. Req *http.Request @@ -26,6 +32,7 @@ type dumpTest struct { WantDump string WantDumpOut string + MustError bool // if true, the test is expected to throw an error NoBody bool // if true, set DumpRequest{,Out} body to false } @@ -203,9 +210,42 @@ var dumpTests = []dumpTest{ "Content-Length: 0\r\n" + "Accept-Encoding: gzip\r\n\r\n", }, + + // Issue 34504: a non-nil Body without ContentLength set should be chunked + { + Req: &http.Request{ + Method: "PUT", + URL: &url.URL{ + Scheme: "http", + Host: "post.tld", + Path: "/test", + }, + ContentLength: 0, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Body: &eofReader{}, + }, + NoBody: true, + WantDumpOut: "PUT /test HTTP/1.1\r\n" + + "Host: post.tld\r\n" + + "User-Agent: Go-http-client/1.1\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Accept-Encoding: gzip\r\n\r\n", + }, } func TestDumpRequest(t *testing.T) { + // Make a copy of dumpTests and add 10 new cases with an empty URL + // to test that no goroutines are leaked. See golang.org/issue/32571. + // 10 seems to be a decent number which always triggers the failure. + dumpTests := dumpTests[:] + for i := 0; i < 10; i++ { + dumpTests = append(dumpTests, dumpTest{ + Req: mustNewRequest("GET", "", nil), + MustError: true, + }) + } numg0 := runtime.NumGoroutine() for i, tt := range dumpTests { if tt.Req != nil && tt.GetReq != nil || tt.Req == nil && tt.GetReq == nil { @@ -250,6 +290,15 @@ func TestDumpRequest(t *testing.T) { } } + if tt.MustError { + req := freshReq(tt) + _, err := DumpRequestOut(req, !tt.NoBody) + if err == nil { + t.Errorf("DumpRequestOut #%d: expected an error, got nil", i) + } + continue + } + if tt.WantDumpOut != "" { req := freshReq(tt) dump, err := DumpRequestOut(req, !tt.NoBody) diff --git a/libgo/go/net/http/httputil/reverseproxy_test.go b/libgo/go/net/http/httputil/reverseproxy_test.go index 7f9dc08..f58e088 100644 --- a/libgo/go/net/http/httputil/reverseproxy_test.go +++ b/libgo/go/net/http/httputil/reverseproxy_test.go @@ -436,7 +436,7 @@ func TestReverseProxyCancelation(t *testing.T) { } if err == nil { // This should be an error like: - // Get http://127.0.0.1:58079: read tcp 127.0.0.1:58079: + // Get "http://127.0.0.1:58079": read tcp 127.0.0.1:58079: // use of closed network connection t.Error("Server.Client().Do() returned nil error; want non-nil error") } diff --git a/libgo/go/net/http/main_test.go b/libgo/go/net/http/main_test.go index 7936fb3..35cc809 100644 --- a/libgo/go/net/http/main_test.go +++ b/libgo/go/net/http/main_test.go @@ -90,6 +90,9 @@ func goroutineLeaked() bool { // (all.bash), but as a serial test otherwise. Using t.Parallel isn't // compatible with the afterTest func in non-short mode. func setParallel(t *testing.T) { + if strings.Contains(t.Name(), "HTTP2") { + http.CondSkipHTTP2(t) + } if testing.Short() { t.Parallel() } @@ -122,7 +125,7 @@ func afterTest(t testing.TB) { ").noteClientGone(": "a closenotifier sender", } var stacks string - for i := 0; i < 4; i++ { + for i := 0; i < 10; i++ { bad = "" stacks = strings.Join(interestingGoroutines(), "\n\n") for substr, what := range badSubstring { diff --git a/libgo/go/net/http/omithttp2.go b/libgo/go/net/http/omithttp2.go new file mode 100644 index 0000000..a0b33e9 --- /dev/null +++ b/libgo/go/net/http/omithttp2.go @@ -0,0 +1,71 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build nethttpomithttp2 + +package http + +import ( + "errors" + "sync" + "time" +) + +func init() { + omitBundledHTTP2 = true +} + +const noHTTP2 = "no bundled HTTP/2" // should never see this + +var http2errRequestCanceled = errors.New("net/http: request canceled") + +var http2goAwayTimeout = 1 * time.Second + +const http2NextProtoTLS = "h2" + +type http2Transport struct { + MaxHeaderListSize uint32 + ConnPool interface{} +} + +func (*http2Transport) RoundTrip(*Request) (*Response, error) { panic(noHTTP2) } +func (*http2Transport) CloseIdleConnections() {} + +type http2erringRoundTripper struct{} + +func (http2erringRoundTripper) RoundTrip(*Request) (*Response, error) { panic(noHTTP2) } + +type http2noDialClientConnPool struct { + http2clientConnPool http2clientConnPool +} + +type http2clientConnPool struct { + mu *sync.Mutex + conns map[string][]struct{} +} + +func http2configureTransport(*Transport) (*http2Transport, error) { panic(noHTTP2) } + +func http2isNoCachedConnError(err error) bool { + _, ok := err.(interface{ IsHTTP2NoCachedConnError() }) + return ok +} + +type http2Server struct { + NewWriteScheduler func() http2WriteScheduler +} + +type http2WriteScheduler interface{} + +func http2NewPriorityWriteScheduler(interface{}) http2WriteScheduler { panic(noHTTP2) } + +func http2ConfigureServer(s *Server, conf *http2Server) error { panic(noHTTP2) } + +var http2ErrNoCachedConn = http2noCachedConnError{} + +type http2noCachedConnError struct{} + +func (http2noCachedConnError) IsHTTP2NoCachedConnError() {} + +func (http2noCachedConnError) Error() string { return "http2: no cached connection was available" } diff --git a/libgo/go/net/http/pprof/pprof.go b/libgo/go/net/http/pprof/pprof.go index 35b3285..a237f58 100644 --- a/libgo/go/net/http/pprof/pprof.go +++ b/libgo/go/net/http/pprof/pprof.go @@ -20,6 +20,9 @@ // log.Println(http.ListenAndServe("localhost:6060", nil)) // }() // +// If you are not using DefaultServeMux, you will have to register handlers +// with the mux you are using. +// // Then use the pprof tool to look at the heap profile: // // go tool pprof http://localhost:6060/debug/pprof/heap diff --git a/libgo/go/net/http/readrequest_test.go b/libgo/go/net/http/readrequest_test.go index 517a818..b227bb6 100644 --- a/libgo/go/net/http/readrequest_test.go +++ b/libgo/go/net/http/readrequest_test.go @@ -133,7 +133,7 @@ var reqTests = []reqTest{ nil, noBodyStr, noTrailer, - "parse ../../../../etc/passwd: invalid URI for request", + `parse "../../../../etc/passwd": invalid URI for request`, }, // Tests missing URL: @@ -143,7 +143,7 @@ var reqTests = []reqTest{ nil, noBodyStr, noTrailer, - "parse : empty url", + `parse "": empty url`, }, // Tests chunked body with trailer: diff --git a/libgo/go/net/http/request.go b/libgo/go/net/http/request.go index 31d6208..8dd9fe1 100644 --- a/libgo/go/net/http/request.go +++ b/libgo/go/net/http/request.go @@ -22,6 +22,7 @@ import ( "net/http/httptrace" "net/textproto" "net/url" + urlpkg "net/url" "strconv" "strings" "sync" @@ -217,9 +218,11 @@ type Request struct { // Transport.DisableKeepAlives were set. Close bool - // For server requests, Host specifies the host on which the URL - // is sought. Per RFC 7230, section 5.4, this is either the value - // of the "Host" header or the host name given in the URL itself. + // For server requests, Host specifies the host on which the + // URL is sought. For HTTP/1 (per RFC 7230, section 5.4), this + // is either the value of the "Host" header or the host name + // given in the URL itself. For HTTP/2, it is the value of the + // ":authority" pseudo-header field. // It may be of the form "host:port". For international domain // names, Host may be in Punycode or Unicode form. Use // golang.org/x/net/idna to convert it to either format if @@ -347,8 +350,8 @@ func (r *Request) Context() context.Context { // sending the request, and reading the response headers and body. // // To create a new request with a context, use NewRequestWithContext. -// To change the context of a request (such as an incoming) you then -// also want to modify to send back out, use Request.Clone. Between +// To change the context of a request, such as an incoming request you +// want to modify before sending back out, use Request.Clone. Between // those two uses, it's rare to need WithContext. func (r *Request) WithContext(ctx context.Context) *Request { if ctx == nil { @@ -763,7 +766,7 @@ func removeZone(host string) string { return host[:j] + host[i:] } -// ParseHTTPVersion parses a HTTP version string. +// ParseHTTPVersion parses an HTTP version string. // "HTTP/1.0" returns (1, 0, true). func ParseHTTPVersion(vers string) (major, minor int, ok bool) { const Big = 1000000 // arbitrary upper bound @@ -848,7 +851,7 @@ func NewRequestWithContext(ctx context.Context, method, url string, body io.Read if ctx == nil { return nil, errors.New("net/http: nil Context") } - u, err := parseURL(url) // Just url.Parse (url is shadowed for godoc). + u, err := urlpkg.Parse(url) if err != nil { return nil, err } @@ -1165,9 +1168,7 @@ func (l *maxBytesReader) Close() error { func copyValues(dst, src url.Values) { for k, vs := range src { - for _, value := range vs { - dst.Add(k, value) - } + dst[k] = append(dst[k], vs...) } } diff --git a/libgo/go/net/http/request_test.go b/libgo/go/net/http/request_test.go index b072f95..42c16d0 100644 --- a/libgo/go/net/http/request_test.go +++ b/libgo/go/net/http/request_test.go @@ -85,35 +85,37 @@ func TestParseFormQueryMethods(t *testing.T) { } } -type stringMap map[string][]string -type parseContentTypeTest struct { - shouldError bool - contentType stringMap -} - -var parseContentTypeTests = []parseContentTypeTest{ - {false, stringMap{"Content-Type": {"text/plain"}}}, - // Empty content type is legal - may be treated as - // application/octet-stream (RFC 7231, section 3.1.1.5) - {false, stringMap{}}, - {true, stringMap{"Content-Type": {"text/plain; boundary="}}}, - {false, stringMap{"Content-Type": {"application/unknown"}}}, -} - func TestParseFormUnknownContentType(t *testing.T) { - for i, test := range parseContentTypeTests { - req := &Request{ - Method: "POST", - Header: Header(test.contentType), - Body: ioutil.NopCloser(strings.NewReader("body")), - } - err := req.ParseForm() - switch { - case err == nil && test.shouldError: - t.Errorf("test %d should have returned error", i) - case err != nil && !test.shouldError: - t.Errorf("test %d should not have returned error, got %v", i, err) - } + for _, test := range []struct { + name string + wantErr string + contentType Header + }{ + {"text", "", Header{"Content-Type": {"text/plain"}}}, + // Empty content type is legal - may be treated as + // application/octet-stream (RFC 7231, section 3.1.1.5) + {"empty", "", Header{}}, + {"boundary", "mime: invalid media parameter", Header{"Content-Type": {"text/plain; boundary="}}}, + {"unknown", "", Header{"Content-Type": {"application/unknown"}}}, + } { + t.Run(test.name, + func(t *testing.T) { + req := &Request{ + Method: "POST", + Header: test.contentType, + Body: ioutil.NopCloser(strings.NewReader("body")), + } + err := req.ParseForm() + switch { + case err == nil && test.wantErr != "": + t.Errorf("unexpected success; want error %q", test.wantErr) + case err != nil && test.wantErr == "": + t.Errorf("want success, got error: %v", err) + case test.wantErr != "" && test.wantErr != fmt.Sprint(err): + t.Errorf("got error %q; want %q", err, test.wantErr) + } + }, + ) } } @@ -826,6 +828,34 @@ func TestWithContextDeepCopiesURL(t *testing.T) { } } +func TestNoPanicOnRoundTripWithBasicAuth_h1(t *testing.T) { + testNoPanicWithBasicAuth(t, h1Mode) +} + +func TestNoPanicOnRoundTripWithBasicAuth_h2(t *testing.T) { + testNoPanicWithBasicAuth(t, h2Mode) +} + +// Issue 34878: verify we don't panic when including basic auth (Go 1.13 regression) +func testNoPanicWithBasicAuth(t *testing.T, h2 bool) { + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) {})) + defer cst.close() + + u, err := url.Parse(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + u.User = url.UserPassword("foo", "bar") + req := &Request{ + URL: u, + Method: "GET", + } + if _, err := cst.c.Do(req); err != nil { + t.Fatalf("Unexpected error: %v", err) + } +} + // verify that NewRequest sets Request.GetBody and that it works func TestNewRequestGetBody(t *testing.T) { tests := []struct { diff --git a/libgo/go/net/http/response_test.go b/libgo/go/net/http/response_test.go index ee7f0d0..0c78df6 100644 --- a/libgo/go/net/http/response_test.go +++ b/libgo/go/net/http/response_test.go @@ -636,6 +636,11 @@ var readResponseCloseInMiddleTests = []struct { {true, true}, } +type readerAndCloser struct { + io.Reader + io.Closer +} + // TestReadResponseCloseInMiddle tests that closing a body after // reading only part of its contents advances the read to the end of // the request, right up until the next request. diff --git a/libgo/go/net/http/roundtrip_js.go b/libgo/go/net/http/roundtrip_js.go index 6331351..4dd9965 100644 --- a/libgo/go/net/http/roundtrip_js.go +++ b/libgo/go/net/http/roundtrip_js.go @@ -41,7 +41,7 @@ const jsFetchCreds = "js.fetch:credentials" // Reference: https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#Parameters const jsFetchRedirect = "js.fetch:redirect" -var useFakeNetwork = js.Global().Get("fetch") == js.Undefined() +var useFakeNetwork = js.Global().Get("fetch").IsUndefined() // RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. func (t *Transport) RoundTrip(req *Request) (*Response, error) { @@ -50,7 +50,7 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { } ac := js.Global().Get("AbortController") - if ac != js.Undefined() { + if !ac.IsUndefined() { // Some browsers that support WASM don't necessarily support // the AbortController. See // https://developer.mozilla.org/en-US/docs/Web/API/AbortController#Browser_compatibility. @@ -74,7 +74,7 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { opt.Set("redirect", h) req.Header.Del(jsFetchRedirect) } - if ac != js.Undefined() { + if !ac.IsUndefined() { opt.Set("signal", ac.Get("signal")) } headers := js.Global().Get("Headers").New() @@ -132,7 +132,7 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { var body io.ReadCloser // The body is undefined when the browser does not support streaming response bodies (Firefox), // and null in certain error cases, i.e. when the request is blocked because of CORS settings. - if b != js.Undefined() && b != js.Null() { + if !b.IsUndefined() && !b.IsNull() { body = &streamReader{stream: b.Call("getReader")} } else { // Fall back to using ArrayBuffer @@ -168,7 +168,7 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { respPromise.Call("then", success, failure) select { case <-req.Context().Done(): - if ac != js.Undefined() { + if !ac.IsUndefined() { // Abort the Fetch request ac.Call("abort") } diff --git a/libgo/go/net/http/serve_test.go b/libgo/go/net/http/serve_test.go index 61adda2..29b9379 100644 --- a/libgo/go/net/http/serve_test.go +++ b/libgo/go/net/http/serve_test.go @@ -10,6 +10,7 @@ import ( "bufio" "bytes" "compress/gzip" + "compress/zlib" "context" "crypto/tls" "encoding/json" @@ -28,10 +29,11 @@ import ( "net/url" "os" "os/exec" + "path/filepath" "reflect" + "regexp" "runtime" "runtime/debug" - "sort" "strconv" "strings" "sync" @@ -1504,6 +1506,7 @@ func TestTLSServer(t *testing.T) { } func TestServeTLS(t *testing.T) { + CondSkipHTTP2(t) // Not parallel: uses global test hooks. defer afterTest(t) defer SetTestHookServerServe(nil) @@ -1654,6 +1657,7 @@ func TestAutomaticHTTP2_ListenAndServe_GetCertificate(t *testing.T) { } func testAutomaticHTTP2_ListenAndServe(t *testing.T, tlsConf *tls.Config) { + CondSkipHTTP2(t) // Not parallel: uses global test hooks. defer afterTest(t) defer SetTestHookServerServe(nil) @@ -2627,7 +2631,7 @@ func TestRedirect(t *testing.T) { // Test that Redirect sets Content-Type header for GET and HEAD requests // and writes a short HTML body, unless the request already has a Content-Type header. -func TestRedirect_contentTypeAndBody(t *testing.T) { +func TestRedirectContentTypeAndBody(t *testing.T) { type ctHeader struct { Values []string } @@ -2906,7 +2910,7 @@ func TestStripPrefix(t *testing.T) { } // https://golang.org/issue/18952. -func TestStripPrefix_notModifyRequest(t *testing.T) { +func TestStripPrefixNotModifyRequest(t *testing.T) { h := StripPrefix("/foo", NotFoundHandler()) req := httptest.NewRequest("GET", "/foo/bar", nil) h.ServeHTTP(httptest.NewRecorder(), req) @@ -4111,14 +4115,49 @@ func TestServerConnState(t *testing.T) { panic("intentional panic") }, } + + // A stateLog is a log of states over the lifetime of a connection. + type stateLog struct { + active net.Conn // The connection for which the log is recorded; set to the first connection seen in StateNew. + got []ConnState + want []ConnState + complete chan<- struct{} // If non-nil, closed when either 'got' is equal to 'want', or 'got' is no longer a prefix of 'want'. + } + activeLog := make(chan *stateLog, 1) + + // wantLog invokes doRequests, then waits for the resulting connection to + // either pass through the sequence of states in want or enter a state outside + // of that sequence. + wantLog := func(doRequests func(), want ...ConnState) { + t.Helper() + complete := make(chan struct{}) + activeLog <- &stateLog{want: want, complete: complete} + + doRequests() + + timer := time.NewTimer(5 * time.Second) + select { + case <-timer.C: + t.Errorf("Timed out waiting for connection to change state.") + case <-complete: + timer.Stop() + } + sl := <-activeLog + if !reflect.DeepEqual(sl.got, sl.want) { + t.Errorf("Request(s) produced unexpected state sequence.\nGot: %v\nWant: %v", sl.got, sl.want) + } + // Don't return sl to activeLog: we don't expect any further states after + // this point, and want to keep the ConnState callback blocked until the + // next call to wantLog. + } + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { handler[r.URL.Path](w, r) })) - defer ts.Close() - - var mu sync.Mutex // guard stateLog and connID - var stateLog = map[int][]ConnState{} - var connID = map[net.Conn]int{} + defer func() { + activeLog <- &stateLog{} // If the test failed, allow any remaining ConnState callbacks to complete. + ts.Close() + }() ts.Config.ErrorLog = log.New(ioutil.Discard, "", 0) ts.Config.ConnState = func(c net.Conn, state ConnState) { @@ -4126,20 +4165,27 @@ func TestServerConnState(t *testing.T) { t.Errorf("nil conn seen in state %s", state) return } - mu.Lock() - defer mu.Unlock() - id, ok := connID[c] - if !ok { - id = len(connID) + 1 - connID[c] = id + sl := <-activeLog + if sl.active == nil && state == StateNew { + sl.active = c + } else if sl.active != c { + t.Errorf("unexpected conn in state %s", state) + activeLog <- sl + return + } + sl.got = append(sl.got, state) + if sl.complete != nil && (len(sl.got) >= len(sl.want) || !reflect.DeepEqual(sl.got, sl.want[:len(sl.got)])) { + close(sl.complete) + sl.complete = nil } - stateLog[id] = append(stateLog[id], state) + activeLog <- sl } - ts.Start() + ts.Start() c := ts.Client() mustGet := func(url string, headers ...string) { + t.Helper() req, err := NewRequest("GET", url, nil) if err != nil { t.Fatal(err) @@ -4160,26 +4206,33 @@ func TestServerConnState(t *testing.T) { } } - mustGet(ts.URL + "/") - mustGet(ts.URL + "/close") + wantLog(func() { + mustGet(ts.URL + "/") + mustGet(ts.URL + "/close") + }, StateNew, StateActive, StateIdle, StateActive, StateClosed) - mustGet(ts.URL + "/") - mustGet(ts.URL+"/", "Connection", "close") + wantLog(func() { + mustGet(ts.URL + "/") + mustGet(ts.URL+"/", "Connection", "close") + }, StateNew, StateActive, StateIdle, StateActive, StateClosed) - mustGet(ts.URL + "/hijack") - mustGet(ts.URL + "/hijack-panic") + wantLog(func() { + mustGet(ts.URL + "/hijack") + }, StateNew, StateActive, StateHijacked) - // New->Closed - { + wantLog(func() { + mustGet(ts.URL + "/hijack-panic") + }, StateNew, StateActive, StateHijacked) + + wantLog(func() { c, err := net.Dial("tcp", ts.Listener.Addr().String()) if err != nil { t.Fatal(err) } c.Close() - } + }, StateNew, StateClosed) - // New->Active->Closed - { + wantLog(func() { c, err := net.Dial("tcp", ts.Listener.Addr().String()) if err != nil { t.Fatal(err) @@ -4189,10 +4242,9 @@ func TestServerConnState(t *testing.T) { } c.Read(make([]byte, 1)) // block until server hangs up on us c.Close() - } + }, StateNew, StateActive, StateClosed) - // New->Idle->Closed - { + wantLog(func() { c, err := net.Dial("tcp", ts.Listener.Addr().String()) if err != nil { t.Fatal(err) @@ -4208,47 +4260,7 @@ func TestServerConnState(t *testing.T) { t.Fatal(err) } c.Close() - } - - want := map[int][]ConnState{ - 1: {StateNew, StateActive, StateIdle, StateActive, StateClosed}, - 2: {StateNew, StateActive, StateIdle, StateActive, StateClosed}, - 3: {StateNew, StateActive, StateHijacked}, - 4: {StateNew, StateActive, StateHijacked}, - 5: {StateNew, StateClosed}, - 6: {StateNew, StateActive, StateClosed}, - 7: {StateNew, StateActive, StateIdle, StateClosed}, - } - logString := func(m map[int][]ConnState) string { - var b bytes.Buffer - var keys []int - for id := range m { - keys = append(keys, id) - } - sort.Ints(keys) - for _, id := range keys { - fmt.Fprintf(&b, "Conn %d: ", id) - for _, s := range m[id] { - fmt.Fprintf(&b, "%s ", s) - } - b.WriteString("\n") - } - return b.String() - } - - for i := 0; i < 5; i++ { - time.Sleep(time.Duration(i) * 50 * time.Millisecond) - mu.Lock() - match := reflect.DeepEqual(stateLog, want) - mu.Unlock() - if match { - return - } - } - - mu.Lock() - t.Errorf("Unexpected events.\nGot log:\n%s\n Want:\n%s\n", logString(stateLog), logString(want)) - mu.Unlock() + }, StateNew, StateActive, StateIdle, StateClosed) } func TestServerKeepAlivesEnabled(t *testing.T) { @@ -4347,7 +4359,7 @@ func TestCloseWrite(t *testing.T) { // This verifies that a handler can Flush and then Hijack. // -// An similar test crashed once during development, but it was only +// A similar test crashed once during development, but it was only // testing this tangentially and temporarily until another TODO was // fixed. // @@ -4755,6 +4767,10 @@ func TestServerValidatesHeaders(t *testing.T) { {"foo\xffbar: foo\r\n", 400}, // binary in header {"foo\x00bar: foo\r\n", 400}, // binary in header {"Foo: " + strings.Repeat("x", 1<<21) + "\r\n", 431}, // header too large + // Spaces between the header key and colon are not allowed. + // See RFC 7230, Section 3.2.4. + {"Foo : bar\r\n", 400}, + {"Foo\t: bar\r\n", 400}, {"foo: foo foo\r\n", 200}, // LWS space is okay {"foo: foo\tfoo\r\n", 200}, // LWS tab is okay @@ -6117,6 +6133,39 @@ func TestServerContextsHTTP2(t *testing.T) { } } +// Issue 35750: check ConnContext not modifying context for other connections +func TestConnContextNotModifyingAllContexts(t *testing.T) { + setParallel(t) + defer afterTest(t) + type connKey struct{} + ts := httptest.NewUnstartedServer(HandlerFunc(func(rw ResponseWriter, r *Request) { + rw.Header().Set("Connection", "close") + })) + ts.Config.ConnContext = func(ctx context.Context, c net.Conn) context.Context { + if got := ctx.Value(connKey{}); got != nil { + t.Errorf("in ConnContext, unexpected context key = %#v", got) + } + return context.WithValue(ctx, connKey{}, "conn") + } + ts.Start() + defer ts.Close() + + var res *Response + var err error + + res, err = ts.Client().Get(ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + + res, err = ts.Client().Get(ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() +} + // Issue 30710: ensure that as per the spec, a server responds // with 501 Not Implemented for unsupported transfer-encodings. func TestUnsupportedTransferEncodingsReturn501(t *testing.T) { @@ -6157,6 +6206,217 @@ func TestUnsupportedTransferEncodingsReturn501(t *testing.T) { } } +func TestContentEncodingNoSniffing_h1(t *testing.T) { + testContentEncodingNoSniffing(t, h1Mode) +} + +func TestContentEncodingNoSniffing_h2(t *testing.T) { + testContentEncodingNoSniffing(t, h2Mode) +} + +// Issue 31753: don't sniff when Content-Encoding is set +func testContentEncodingNoSniffing(t *testing.T, h2 bool) { + setParallel(t) + defer afterTest(t) + + type setting struct { + name string + body []byte + + // setting contentEncoding as an interface instead of a string + // directly, so as to differentiate between 3 states: + // unset, empty string "" and set string "foo/bar". + contentEncoding interface{} + wantContentType string + } + + settings := []*setting{ + { + name: "gzip content-encoding, gzipped", // don't sniff. + contentEncoding: "application/gzip", + wantContentType: "", + body: func() []byte { + buf := new(bytes.Buffer) + gzw := gzip.NewWriter(buf) + gzw.Write([]byte("doctype html><p>Hello</p>")) + gzw.Close() + return buf.Bytes() + }(), + }, + { + name: "zlib content-encoding, zlibbed", // don't sniff. + contentEncoding: "application/zlib", + wantContentType: "", + body: func() []byte { + buf := new(bytes.Buffer) + zw := zlib.NewWriter(buf) + zw.Write([]byte("doctype html><p>Hello</p>")) + zw.Close() + return buf.Bytes() + }(), + }, + { + name: "no content-encoding", // must sniff. + wantContentType: "application/x-gzip", + body: func() []byte { + buf := new(bytes.Buffer) + gzw := gzip.NewWriter(buf) + gzw.Write([]byte("doctype html><p>Hello</p>")) + gzw.Close() + return buf.Bytes() + }(), + }, + { + name: "phony content-encoding", // don't sniff. + contentEncoding: "foo/bar", + body: []byte("doctype html><p>Hello</p>"), + }, + { + name: "empty but set content-encoding", + contentEncoding: "", + wantContentType: "audio/mpeg", + body: []byte("ID3"), + }, + } + + for _, tt := range settings { + t.Run(tt.name, func(t *testing.T) { + cst := newClientServerTest(t, h2, HandlerFunc(func(rw ResponseWriter, r *Request) { + if tt.contentEncoding != nil { + rw.Header().Set("Content-Encoding", tt.contentEncoding.(string)) + } + rw.Write(tt.body) + })) + defer cst.close() + + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatalf("Failed to fetch URL: %v", err) + } + defer res.Body.Close() + + if g, w := res.Header.Get("Content-Encoding"), tt.contentEncoding; g != w { + if w != nil { // The case where contentEncoding was set explicitly. + t.Errorf("Content-Encoding mismatch\n\tgot: %q\n\twant: %q", g, w) + } else if g != "" { // "" should be the equivalent when the contentEncoding is unset. + t.Errorf("Unexpected Content-Encoding %q", g) + } + } + + if g, w := res.Header.Get("Content-Type"), tt.wantContentType; g != w { + t.Errorf("Content-Type mismatch\n\tgot: %q\n\twant: %q", g, w) + } + }) + } +} + +// Issue 30803: ensure that TimeoutHandler logs spurious +// WriteHeader calls, for consistency with other Handlers. +func TestTimeoutHandlerSuperfluousLogs(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + setParallel(t) + defer afterTest(t) + + pc, curFile, _, _ := runtime.Caller(0) + curFileBaseName := filepath.Base(curFile) + testFuncName := runtime.FuncForPC(pc).Name() + + timeoutMsg := "timed out here!" + + tests := []struct { + name string + mustTimeout bool + wantResp string + }{ + { + name: "return before timeout", + wantResp: "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n", + }, + { + name: "return after timeout", + mustTimeout: true, + wantResp: fmt.Sprintf("HTTP/1.1 503 Service Unavailable\r\nContent-Length: %d\r\n\r\n%s", + len(timeoutMsg), timeoutMsg), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + exitHandler := make(chan bool, 1) + defer close(exitHandler) + lastLine := make(chan int, 1) + + sh := HandlerFunc(func(w ResponseWriter, r *Request) { + w.WriteHeader(404) + w.WriteHeader(404) + w.WriteHeader(404) + w.WriteHeader(404) + _, _, line, _ := runtime.Caller(0) + lastLine <- line + <-exitHandler + }) + + if !tt.mustTimeout { + exitHandler <- true + } + + logBuf := new(bytes.Buffer) + srvLog := log.New(logBuf, "", 0) + // When expecting to timeout, we'll keep the duration short. + dur := 20 * time.Millisecond + if !tt.mustTimeout { + // Otherwise, make it arbitrarily long to reduce the risk of flakes. + dur = 10 * time.Second + } + th := TimeoutHandler(sh, dur, timeoutMsg) + cst := newClientServerTest(t, h1Mode /* the test is protocol-agnostic */, th, optWithServerLog(srvLog)) + defer cst.close() + + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // Deliberately removing the "Date" header since it is highly ephemeral + // and will cause failure if we try to match it exactly. + res.Header.Del("Date") + res.Header.Del("Content-Type") + + // Match the response. + blob, _ := httputil.DumpResponse(res, true) + if g, w := string(blob), tt.wantResp; g != w { + t.Errorf("Response mismatch\nGot\n%q\n\nWant\n%q", g, w) + } + + // Given 4 w.WriteHeader calls, only the first one is valid + // and the rest should be reported as the 3 spurious logs. + logEntries := strings.Split(strings.TrimSpace(logBuf.String()), "\n") + if g, w := len(logEntries), 3; g != w { + blob, _ := json.MarshalIndent(logEntries, "", " ") + t.Fatalf("Server logs count mismatch\ngot %d, want %d\n\nGot\n%s\n", g, w, blob) + } + + lastSpuriousLine := <-lastLine + firstSpuriousLine := lastSpuriousLine - 3 + // Now ensure that the regexes match exactly. + // "http: superfluous response.WriteHeader call from <fn>.func\d.\d (<curFile>:lastSpuriousLine-[1, 3]" + for i, logEntry := range logEntries { + wantLine := firstSpuriousLine + i + pat := fmt.Sprintf("^http: superfluous response.WriteHeader call from %s.func\\d+.\\d+ \\(%s:%d\\)$", + testFuncName, curFileBaseName, wantLine) + re := regexp.MustCompile(pat) + if !re.MatchString(logEntry) { + t.Errorf("Log entry mismatch\n\t%s\ndoes not match\n\t%s", logEntry, pat) + } + } + }) + } +} + // fetchWireResponse is a helper for dialing to host, // sending http1ReqBody as the payload and retrieving // the response as it was sent on the wire. diff --git a/libgo/go/net/http/server.go b/libgo/go/net/http/server.go index aaf7b68..77329b2 100644 --- a/libgo/go/net/http/server.go +++ b/libgo/go/net/http/server.go @@ -19,6 +19,7 @@ import ( "net" "net/textproto" "net/url" + urlpkg "net/url" "os" "path" "runtime" @@ -1384,7 +1385,12 @@ func (cw *chunkWriter) writeHeader(p []byte) { if bodyAllowedForStatus(code) { // If no content type, apply sniffing algorithm to body. _, haveType := header["Content-Type"] - if !haveType && !hasTE && len(p) > 0 { + + // If the Content-Encoding was set and is non-blank, + // we shouldn't sniff the body. See Issue 31753. + ce := header.Get("Content-Encoding") + hasCE := len(ce) > 0 + if !hasCE && !haveType && !hasTE && len(p) > 0 { setHeader.contentType = DetectContentType(p) } } else { @@ -1696,11 +1702,10 @@ func (c *conn) closeWriteAndWait() { time.Sleep(rstAvoidanceDelay) } -// validNPN reports whether the proto is not a blacklisted Next -// Protocol Negotiation protocol. Empty and built-in protocol types -// are blacklisted and can't be overridden with alternate -// implementations. -func validNPN(proto string) bool { +// validNextProto reports whether the proto is not a blacklisted ALPN +// protocol name. Empty and built-in protocol types are blacklisted +// and can't be overridden with alternate implementations. +func validNextProto(proto string) bool { switch proto { case "", "http/1.1", "http/1.0": return false @@ -1799,9 +1804,9 @@ func (c *conn) serve(ctx context.Context) { } c.tlsState = new(tls.ConnectionState) *c.tlsState = tlsConn.ConnectionState() - if proto := c.tlsState.NegotiatedProtocol; validNPN(proto) { + if proto := c.tlsState.NegotiatedProtocol; validNextProto(proto) { if fn := c.server.TLSNextProto[proto]; fn != nil { - h := initNPNRequest{ctx, tlsConn, serverHandler{c.server}} + h := initALPNRequest{ctx, tlsConn, serverHandler{c.server}} fn(c.server, tlsConn, h) } return @@ -2066,8 +2071,7 @@ func StripPrefix(prefix string, h Handler) Handler { // Setting the Content-Type header to any value, including nil, // disables that behavior. func Redirect(w ResponseWriter, r *Request, url string, code int) { - // parseURL is just url.Parse (url is shadowed for godoc). - if u, err := parseURL(url); err == nil { + if u, err := urlpkg.Parse(url); err == nil { // If url was relative, make its path absolute by // combining with request path. // The client would probably do this for us, @@ -2121,10 +2125,6 @@ func Redirect(w ResponseWriter, r *Request, url string, code int) { } } -// parseURL is just url.Parse. It exists only so that url.Parse can be called -// in places where url is shadowed for godoc. See https://golang.org/cl/49930. -var parseURL = url.Parse - var htmlReplacer = strings.NewReplacer( "&", "&", "<", "<", @@ -2493,7 +2493,12 @@ func ServeTLS(l net.Listener, handler Handler, certFile, keyFile string) error { // A Server defines parameters for running an HTTP server. // The zero value for Server is a valid configuration. type Server struct { - Addr string // TCP address to listen on, ":http" if empty + // Addr optionally specifies the TCP address for the server to listen on, + // in the form "host:port". If empty, ":http" (port 80) is used. + // The service names are defined in RFC 6335 and assigned by IANA. + // See net.Dial for details of the address format. + Addr string + Handler Handler // handler to invoke, http.DefaultServeMux if nil // TLSConfig optionally provides a TLS configuration for use @@ -2542,7 +2547,7 @@ type Server struct { MaxHeaderBytes int // TLSNextProto optionally specifies a function to take over - // ownership of the provided TLS connection when an NPN/ALPN + // ownership of the provided TLS connection when an ALPN // protocol upgrade has occurred. The map key is the protocol // name negotiated. The Handler argument should be used to // handle HTTP requests and will initialize the Request's TLS @@ -2692,7 +2697,7 @@ func (srv *Server) Shutdown(ctx context.Context) error { // RegisterOnShutdown registers a function to call on Shutdown. // This can be used to gracefully shutdown connections that have -// undergone NPN/ALPN protocol upgrade or that have been hijacked. +// undergone ALPN protocol upgrade or that have been hijacked. // This function should start protocol-specific graceful shutdown, // but should not wait for shutdown to complete. func (srv *Server) RegisterOnShutdown(f func()) { @@ -2886,8 +2891,6 @@ func (srv *Server) Serve(l net.Listener) error { } defer srv.trackListener(&l, false) - var tempDelay time.Duration // how long to sleep on accept failure - baseCtx := context.Background() if srv.BaseContext != nil { baseCtx = srv.BaseContext(origListener) @@ -2896,16 +2899,18 @@ func (srv *Server) Serve(l net.Listener) error { } } + var tempDelay time.Duration // how long to sleep on accept failure + ctx := context.WithValue(baseCtx, ServerContextKey, srv) for { - rw, e := l.Accept() - if e != nil { + rw, err := l.Accept() + if err != nil { select { case <-srv.getDoneChan(): return ErrServerClosed default: } - if ne, ok := e.(net.Error); ok && ne.Temporary() { + if ne, ok := err.(net.Error); ok && ne.Temporary() { if tempDelay == 0 { tempDelay = 5 * time.Millisecond } else { @@ -2914,22 +2919,23 @@ func (srv *Server) Serve(l net.Listener) error { if max := 1 * time.Second; tempDelay > max { tempDelay = max } - srv.logf("http: Accept error: %v; retrying in %v", e, tempDelay) + srv.logf("http: Accept error: %v; retrying in %v", err, tempDelay) time.Sleep(tempDelay) continue } - return e + return err } + connCtx := ctx if cc := srv.ConnContext; cc != nil { - ctx = cc(ctx, rw) - if ctx == nil { + connCtx = cc(connCtx, rw) + if connCtx == nil { panic("ConnContext returned nil") } } tempDelay = 0 c := srv.newConn(rw) c.setState(c.rwc, StateNew) // before Serve can return - go c.serve(ctx) + go c.serve(connCtx) } } @@ -3160,7 +3166,7 @@ func (srv *Server) onceSetNextProtoDefaults_Serve() { // configured otherwise. (by setting srv.TLSNextProto non-nil) // It must only be called via srv.nextProtoOnce (use srv.setupHTTP2_*). func (srv *Server) onceSetNextProtoDefaults() { - if strings.Contains(os.Getenv("GODEBUG"), "http2server=0") { + if omitBundledHTTP2 || strings.Contains(os.Getenv("GODEBUG"), "http2server=0") { return } // Enable HTTP/2 by default if the user hasn't otherwise @@ -3182,8 +3188,8 @@ func (srv *Server) onceSetNextProtoDefaults() { // After such a timeout, writes by h to its ResponseWriter will return // ErrHandlerTimeout. // -// TimeoutHandler supports the Flusher and Pusher interfaces but does not -// support the Hijacker interface. +// TimeoutHandler supports the Pusher interface but does not support +// the Hijacker or Flusher interfaces. func TimeoutHandler(h Handler, dt time.Duration, msg string) Handler { return &timeoutHandler{ handler: h, @@ -3223,8 +3229,9 @@ func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) { r = r.WithContext(ctx) done := make(chan struct{}) tw := &timeoutWriter{ - w: w, - h: make(Header), + w: w, + h: make(Header), + req: r, } panicChan := make(chan interface{}, 1) go func() { @@ -3264,6 +3271,7 @@ type timeoutWriter struct { w ResponseWriter h Header wbuf bytes.Buffer + req *Request mu sync.Mutex timedOut bool @@ -3272,7 +3280,6 @@ type timeoutWriter struct { } var _ Pusher = (*timeoutWriter)(nil) -var _ Flusher = (*timeoutWriter)(nil) // Push implements the Pusher interface. func (tw *timeoutWriter) Push(target string, opts *PushOptions) error { @@ -3282,14 +3289,6 @@ func (tw *timeoutWriter) Push(target string, opts *PushOptions) error { return ErrNotSupported } -// Flush implements the Flusher interface. -func (tw *timeoutWriter) Flush() { - f, ok := tw.w.(Flusher) - if ok { - f.Flush() - } -} - func (tw *timeoutWriter) Header() Header { return tw.h } func (tw *timeoutWriter) Write(p []byte) (int, error) { @@ -3299,24 +3298,32 @@ func (tw *timeoutWriter) Write(p []byte) (int, error) { return 0, ErrHandlerTimeout } if !tw.wroteHeader { - tw.writeHeader(StatusOK) + tw.writeHeaderLocked(StatusOK) } return tw.wbuf.Write(p) } -func (tw *timeoutWriter) WriteHeader(code int) { +func (tw *timeoutWriter) writeHeaderLocked(code int) { checkWriteHeaderCode(code) - tw.mu.Lock() - defer tw.mu.Unlock() - if tw.timedOut || tw.wroteHeader { + + switch { + case tw.timedOut: return + case tw.wroteHeader: + if tw.req != nil { + caller := relevantCaller() + logf(tw.req, "http: superfluous response.WriteHeader call from %s (%s:%d)", caller.Function, path.Base(caller.File), caller.Line) + } + default: + tw.wroteHeader = true + tw.code = code } - tw.writeHeader(code) } -func (tw *timeoutWriter) writeHeader(code int) { - tw.wroteHeader = true - tw.code = code +func (tw *timeoutWriter) WriteHeader(code int) { + tw.mu.Lock() + defer tw.mu.Unlock() + tw.writeHeaderLocked(code) } // onceCloseListener wraps a net.Listener, protecting it from @@ -3350,10 +3357,10 @@ func (globalOptionsHandler) ServeHTTP(w ResponseWriter, r *Request) { } } -// initNPNRequest is an HTTP handler that initializes certain +// initALPNRequest is an HTTP handler that initializes certain // uninitialized fields in its *Request. Such partially-initialized -// Requests come from NPN protocol handlers. -type initNPNRequest struct { +// Requests come from ALPN protocol handlers. +type initALPNRequest struct { ctx context.Context c *tls.Conn h serverHandler @@ -3363,9 +3370,9 @@ type initNPNRequest struct { // recognized by x/net/http2 to pass down a context; the TLSNextProto // API predates context support so we shoehorn through the only // interface we have available. -func (h initNPNRequest) BaseContext() context.Context { return h.ctx } +func (h initALPNRequest) BaseContext() context.Context { return h.ctx } -func (h initNPNRequest) ServeHTTP(rw ResponseWriter, req *Request) { +func (h initALPNRequest) ServeHTTP(rw ResponseWriter, req *Request) { if req.TLS == nil { req.TLS = &tls.ConnectionState{} *req.TLS = h.c.ConnectionState() diff --git a/libgo/go/net/http/socks_bundle.go b/libgo/go/net/http/socks_bundle.go index d22d636..e446669 100644 --- a/libgo/go/net/http/socks_bundle.go +++ b/libgo/go/net/http/socks_bundle.go @@ -283,7 +283,7 @@ type socksDialer struct { // establishing the transport connection. ProxyDial func(context.Context, string, string) (net.Conn, error) - // AuthMethods specifies the list of request authention + // AuthMethods specifies the list of request authentication // methods. // If empty, SOCKS client requests only AuthMethodNotRequired. AuthMethods []socksAuthMethod diff --git a/libgo/go/net/http/transfer.go b/libgo/go/net/http/transfer.go index 2e01a07..1d6a987 100644 --- a/libgo/go/net/http/transfer.go +++ b/libgo/go/net/http/transfer.go @@ -7,6 +7,7 @@ package http import ( "bufio" "bytes" + "compress/gzip" "errors" "fmt" "io" @@ -466,6 +467,34 @@ func suppressedHeaders(status int) []string { return nil } +// proxyingReadCloser is a composite type that accepts and proxies +// io.Read and io.Close calls to its respective Reader and Closer. +// +// It is composed of: +// a) a top-level reader e.g. the result of decompression +// b) a symbolic Closer e.g. the result of decompression, the +// original body and the connection itself. +type proxyingReadCloser struct { + io.Reader + io.Closer +} + +// multiCloser implements io.Closer and allows a bunch of io.Closer values +// to all be closed once. +// Example usage is with proxyingReadCloser if we are decompressing a response +// body on the fly and would like to close both *gzip.Reader and underlying body. +type multiCloser []io.Closer + +func (mc multiCloser) Close() error { + var err error + for _, c := range mc { + if err1 := c.Close(); err1 != nil && err == nil { + err = err1 + } + } + return err +} + // msg is *Request or *Response. func readTransfer(msg interface{}, r *bufio.Reader) (err error) { t := &transferReader{RequestMethod: "GET"} @@ -543,7 +572,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { // Prepare body reader. ContentLength < 0 means chunked encoding // or close connection when finished, since multipart is not supported yet switch { - case chunked(t.TransferEncoding): + case chunked(t.TransferEncoding) || implicitlyChunked(t.TransferEncoding): if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) { t.Body = NoBody } else { @@ -564,6 +593,21 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { } } + // Finally if "gzip" was one of the requested transfer-encodings, + // we'll unzip the concatenated body/payload of the request. + // TODO: As we support more transfer-encodings, extract + // this code and apply the un-codings in reverse. + if t.Body != NoBody && gzipped(t.TransferEncoding) { + zr, err := gzip.NewReader(t.Body) + if err != nil { + return fmt.Errorf("http: failed to gunzip body: %v", err) + } + t.Body = &proxyingReadCloser{ + Reader: zr, + Closer: multiCloser{zr, t.Body}, + } + } + // Unify output switch rr := msg.(type) { case *Request: @@ -583,8 +627,41 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { return nil } -// Checks whether chunked is part of the encodings stack -func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } +// Checks whether chunked is the last part of the encodings stack +func chunked(te []string) bool { return len(te) > 0 && te[len(te)-1] == "chunked" } + +// implicitlyChunked is a helper to check for implicity of chunked, because +// RFC 7230 Section 3.3.1 says that the sender MUST apply chunked as the final +// payload body to ensure that the message is framed for both the request +// and the body. Since "identity" is incompatible with any other transformational +// encoding cannot co-exist, the presence of "identity" will cause implicitlyChunked +// to return false. +func implicitlyChunked(te []string) bool { + if len(te) == 0 { // No transfer-encodings passed in, so not implicitly chunked. + return false + } + for _, tei := range te { + if tei == "identity" { + return false + } + } + return true +} + +func isGzipTransferEncoding(tei string) bool { + // RFC 7230 4.2.3 requests that "x-gzip" SHOULD be considered the same as "gzip". + return tei == "gzip" || tei == "x-gzip" +} + +// Checks where either of "gzip" or "x-gzip" are contained in transfer encodings. +func gzipped(te []string) bool { + for _, tei := range te { + if isGzipTransferEncoding(tei) { + return true + } + } + return false +} // Checks whether the encoding is explicitly "identity". func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } @@ -620,25 +697,47 @@ func (t *transferReader) fixTransferEncoding() error { encodings := strings.Split(raw[0], ",") te := make([]string, 0, len(encodings)) - // TODO: Even though we only support "identity" and "chunked" - // encodings, the loop below is designed with foresight. One - // invariant that must be maintained is that, if present, - // chunked encoding must always come first. - for _, encoding := range encodings { + + // When adding new encodings, please maintain the invariant: + // if chunked encoding is present, it must always + // come last and it must be applied only once. + // See RFC 7230 Section 3.3.1 Transfer-Encoding. + for i, encoding := range encodings { encoding = strings.ToLower(strings.TrimSpace(encoding)) - // "identity" encoding is not recorded + if encoding == "identity" { + // "identity" should not be mixed with other transfer-encodings/compressions + // because it means "no compression, no transformation". + if len(encodings) != 1 { + return &badStringError{`"identity" when present must be the only transfer encoding`, strings.Join(encodings, ",")} + } + // "identity" is not recorded. break } - if encoding != "chunked" { + + switch { + case encoding == "chunked": + // "chunked" MUST ALWAYS be the last + // encoding as per the loop invariant. + // That is: + // Invalid: [chunked, gzip] + // Valid: [gzip, chunked] + if i+1 != len(encodings) { + return &badStringError{"chunked must be applied only once, as the last encoding", strings.Join(encodings, ",")} + } + // Supported otherwise. + + case isGzipTransferEncoding(encoding): + // Supported + + default: return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", encoding)} } + te = te[0 : len(te)+1] te[len(te)-1] = encoding } - if len(te) > 1 { - return &badStringError{"too many transfer encodings", strings.Join(te, ",")} - } + if len(te) > 0 { // RFC 7230 3.3.2 says "A sender MUST NOT send a // Content-Length header field in any message that diff --git a/libgo/go/net/http/transfer_test.go b/libgo/go/net/http/transfer_test.go index 65009ee..a8ce2d3 100644 --- a/libgo/go/net/http/transfer_test.go +++ b/libgo/go/net/http/transfer_test.go @@ -7,6 +7,7 @@ package http import ( "bufio" "bytes" + "compress/gzip" "crypto/rand" "fmt" "io" @@ -61,7 +62,6 @@ func TestFinalChunkedBodyReadEOF(t *testing.T) { buf := make([]byte, len(want)) n, err := res.Body.Read(buf) if n != len(want) || err != io.EOF { - t.Logf("body = %#v", res.Body) t.Errorf("Read = %v, %v; want %d, EOF", n, err, len(want)) } if string(buf) != want { @@ -290,7 +290,7 @@ func TestFixTransferEncoding(t *testing.T) { }, { hdr: Header{"Transfer-Encoding": {"chunked, chunked", "identity", "chunked"}}, - wantErr: &badStringError{"too many transfer encodings", "chunked,chunked"}, + wantErr: &badStringError{"chunked must be applied only once, as the last encoding", "chunked, chunked"}, }, { hdr: Header{"Transfer-Encoding": {"chunked"}}, @@ -310,3 +310,283 @@ func TestFixTransferEncoding(t *testing.T) { } } } + +func gzipIt(s string) string { + buf := new(bytes.Buffer) + gw := gzip.NewWriter(buf) + gw.Write([]byte(s)) + gw.Close() + return buf.String() +} + +func TestUnitTestProxyingReadCloserClosesBody(t *testing.T) { + var checker closeChecker + buf := new(bytes.Buffer) + buf.WriteString("Hello, Gophers!") + prc := &proxyingReadCloser{ + Reader: buf, + Closer: &checker, + } + prc.Close() + + read, err := ioutil.ReadAll(prc) + if err != nil { + t.Fatalf("Read error: %v", err) + } + if g, w := string(read), "Hello, Gophers!"; g != w { + t.Errorf("Read mismatch: got %q want %q", g, w) + } + + if checker.closed != true { + t.Fatal("closeChecker.Close was never invoked") + } +} + +func TestGzipTransferEncoding_request(t *testing.T) { + helloWorldGzipped := gzipIt("Hello, World!") + + tests := []struct { + payload string + wantErr string + wantBody string + }{ + + { + // The case of "chunked" properly applied as the last encoding + // and a gzipped request payload that is streamed in 3 parts. + payload: `POST / HTTP/1.1 +Host: golang.org +Transfer-Encoding: gzip, chunked +Content-Type: text/html; charset=UTF-8 + +` + fmt.Sprintf("%02x\r\n%s\r\n%02x\r\n%s\r\n%02x\r\n%s\r\n0\r\n\r\n", + 3, helloWorldGzipped[:3], + 5, helloWorldGzipped[3:8], + len(helloWorldGzipped)-8, helloWorldGzipped[8:]), + wantBody: `Hello, World!`, + }, + + { + // The request specifies "Transfer-Encoding: chunked" so its body must be left untouched. + payload: `PUT / HTTP/1.1 +Host: golang.org +Transfer-Encoding: chunked +Connection: close +Content-Type: text/html; charset=UTF-8 + +` + fmt.Sprintf("%0x\r\n%s\r\n0\r\n\r\n", len(helloWorldGzipped), helloWorldGzipped), + // We want that payload as it was sent. + wantBody: helloWorldGzipped, + }, + + { + // Valid request, the body doesn't have "Transfer-Encoding: chunked" but implicitly encoded + // for chunking as per the advisory from RFC 7230 3.3.1 which advises for cases where. + payload: `POST / HTTP/1.1 +Host: localhost +Transfer-Encoding: gzip +Content-Type: text/html; charset=UTF-8 + +` + fmt.Sprintf("%0x\r\n%s\r\n0\r\n\r\n", len(helloWorldGzipped), helloWorldGzipped), + wantBody: `Hello, World!`, + }, + + { + // Invalid request, the body isn't chunked nor is the connection terminated immediately + // hence invalid as per the advisory from RFC 7230 3.3.1 which advises for cases where + // a Transfer-Encoding that isn't finally chunked is provided. + payload: `PUT / HTTP/1.1 +Host: golang.org +Transfer-Encoding: gzip +Content-Length: 0 +Connection: close +Content-Type: text/html; charset=UTF-8 + +`, + wantErr: `EOF`, + }, + + { + // The case of chunked applied before another encoding. + payload: `PUT / HTTP/1.1 +Location: golang.org +Transfer-Encoding: chunked, gzip +Content-Length: 0 +Connection: close +Content-Type: text/html; charset=UTF-8 + +`, + wantErr: `chunked must be applied only once, as the last encoding "chunked, gzip"`, + }, + + { + // The case of chunked properly applied as the + // last encoding BUT with a bad "Content-Length". + payload: `POST / HTTP/1.1 +Host: golang.org +Transfer-Encoding: gzip, chunked +Content-Length: 10 +Connection: close +Content-Type: text/html; charset=UTF-8 + +` + "0\r\n\r\n", + wantErr: "EOF", + }, + } + + for i, tt := range tests { + req, err := ReadRequest(bufio.NewReader(strings.NewReader(tt.payload))) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("test %d. Error mismatch\nGot: %v\nWant: %s", i, err, tt.wantErr) + } + continue + } + + if err != nil { + t.Errorf("test %d. Unexpected ReadRequest error: %v\nPayload:\n%s", i, err, tt.payload) + continue + } + + got, err := ioutil.ReadAll(req.Body) + req.Body.Close() + if err != nil { + t.Errorf("test %d. Failed to read response body: %v", i, err) + } + if g, w := string(got), tt.wantBody; g != w { + t.Errorf("test %d. Request body mimsatch\nGot:\n%s\n\nWant:\n%s", i, g, w) + } + } +} + +func TestGzipTransferEncoding_response(t *testing.T) { + helloWorldGzipped := gzipIt("Hello, World!") + + tests := []struct { + payload string + wantErr string + wantBody string + }{ + + { + // The case of "chunked" properly applied as the last encoding + // and a gzipped payload that is streamed in 3 parts. + payload: `HTTP/1.1 302 Found +Location: https://golang.org/ +Transfer-Encoding: gzip, chunked +Connection: close +Content-Type: text/html; charset=UTF-8 + +` + fmt.Sprintf("%02x\r\n%s\r\n%02x\r\n%s\r\n%02x\r\n%s\r\n0\r\n\r\n", + 3, helloWorldGzipped[:3], + 5, helloWorldGzipped[3:8], + len(helloWorldGzipped)-8, helloWorldGzipped[8:]), + wantBody: `Hello, World!`, + }, + + { + // The response specifies "Transfer-Encoding: chunked" so response body must be left untouched. + payload: `HTTP/1.1 302 Found +Location: https://golang.org/ +Transfer-Encoding: chunked +Connection: close +Content-Type: text/html; charset=UTF-8 + +` + fmt.Sprintf("%0x\r\n%s\r\n0\r\n\r\n", len(helloWorldGzipped), helloWorldGzipped), + // We want that payload as it was sent. + wantBody: helloWorldGzipped, + }, + + { + // Valid response, the body doesn't have "Transfer-Encoding: chunked" but implicitly encoded + // for chunking as per the advisory from RFC 7230 3.3.1 which advises for cases where. + payload: `HTTP/1.1 302 Found +Location: https://golang.org/ +Transfer-Encoding: gzip +Connection: close +Content-Type: text/html; charset=UTF-8 + +` + fmt.Sprintf("%0x\r\n%s\r\n0\r\n\r\n", len(helloWorldGzipped), helloWorldGzipped), + wantBody: `Hello, World!`, + }, + + { + // Invalid response, the body isn't chunked nor is the connection terminated immediately + // hence invalid as per the advisory from RFC 7230 3.3.1 which advises for cases where + // a Transfer-Encoding that isn't finally chunked is provided. + payload: `HTTP/1.1 302 Found +Location: https://golang.org/ +Transfer-Encoding: gzip +Content-Length: 0 +Connection: close +Content-Type: text/html; charset=UTF-8 + +`, + wantErr: `EOF`, + }, + + { + // The case of chunked applied before another encoding. + payload: `HTTP/1.1 302 Found +Location: https://golang.org/ +Transfer-Encoding: chunked, gzip +Content-Length: 0 +Connection: close +Content-Type: text/html; charset=UTF-8 + +`, + wantErr: `chunked must be applied only once, as the last encoding "chunked, gzip"`, + }, + + { + // The case of chunked properly applied as the + // last encoding BUT with a bad "Content-Length". + payload: `HTTP/1.1 302 Found +Location: https://golang.org/ +Transfer-Encoding: gzip, chunked +Content-Length: 10 +Connection: close +Content-Type: text/html; charset=UTF-8 + +` + "0\r\n\r\n", + wantErr: "EOF", + }, + + { + // Including "identity" more than once. + payload: `HTTP/1.1 200 OK +Location: https://golang.org/ +Transfer-Encoding: identity, identity +Content-Length: 0 +Connection: close +Content-Type: text/html; charset=UTF-8 + +` + "0\r\n\r\n", + wantErr: `"identity" when present must be the only transfer encoding "identity, identity"`, + }, + } + + for i, tt := range tests { + res, err := ReadResponse(bufio.NewReader(strings.NewReader(tt.payload)), nil) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("test %d. Error mismatch\nGot: %v\nWant: %s", i, err, tt.wantErr) + } + continue + } + + if err != nil { + t.Errorf("test %d. Unexpected ReadResponse error: %v\nPayload:\n%s", i, err, tt.payload) + continue + } + + got, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + t.Errorf("test %d. Failed to read response body: %v", i, err) + } + if g, w := string(got), tt.wantBody; g != w { + t.Errorf("test %d. Response body mimsatch\nGot:\n%s\n\nWant:\n%s", i, g, w) + } + } +} diff --git a/libgo/go/net/http/transport.go b/libgo/go/net/http/transport.go index ee279877..64d8510 100644 --- a/libgo/go/net/http/transport.go +++ b/libgo/go/net/http/transport.go @@ -89,7 +89,7 @@ const DefaultMaxIdleConnsPerHost = 2 // Request.GetBody defined. HTTP requests are considered idempotent if // they have HTTP methods GET, HEAD, OPTIONS, or TRACE; or if their // Header map contains an "Idempotency-Key" or "X-Idempotency-Key" -// entry. If the idempotency key value is an zero-length slice, the +// entry. If the idempotency key value is a zero-length slice, the // request is treated as idempotent but the header is not sent on the // wire. type Transport struct { @@ -142,15 +142,24 @@ type Transport struct { // If both are set, DialContext takes priority. Dial func(network, addr string) (net.Conn, error) - // DialTLS specifies an optional dial function for creating + // DialTLSContext specifies an optional dial function for creating // TLS connections for non-proxied HTTPS requests. // - // If DialTLS is nil, Dial and TLSClientConfig are used. + // If DialTLSContext is nil (and the deprecated DialTLS below is also nil), + // DialContext and TLSClientConfig are used. // - // If DialTLS is set, the Dial hook is not used for HTTPS + // If DialTLSContext is set, the Dial and DialContext hooks are not used for HTTPS // requests and the TLSClientConfig and TLSHandshakeTimeout // are ignored. The returned net.Conn is assumed to already be // past the TLS handshake. + DialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error) + + // DialTLS specifies an optional dial function for creating + // TLS connections for non-proxied HTTPS requests. + // + // Deprecated: Use DialTLSContext instead, which allows the transport + // to cancel dials as soon as they are no longer needed. + // If both are set, DialTLSContext takes priority. DialTLS func(network, addr string) (net.Conn, error) // TLSClientConfig specifies the TLS configuration to use with @@ -218,7 +227,7 @@ type Transport struct { ExpectContinueTimeout time.Duration // TLSNextProto specifies how the Transport switches to an - // alternate protocol (such as HTTP/2) after a TLS NPN/ALPN + // alternate protocol (such as HTTP/2) after a TLS ALPN // protocol negotiation. If Transport dials an TLS connection // with a non-empty protocol name and TLSNextProto contains a // map entry for that key (such as "h2"), then the func is @@ -286,7 +295,7 @@ func (t *Transport) Clone() *Transport { DialContext: t.DialContext, Dial: t.Dial, DialTLS: t.DialTLS, - TLSClientConfig: t.TLSClientConfig.Clone(), + DialTLSContext: t.DialTLSContext, TLSHandshakeTimeout: t.TLSHandshakeTimeout, DisableKeepAlives: t.DisableKeepAlives, DisableCompression: t.DisableCompression, @@ -302,6 +311,9 @@ func (t *Transport) Clone() *Transport { WriteBufferSize: t.WriteBufferSize, ReadBufferSize: t.ReadBufferSize, } + if t.TLSClientConfig != nil { + t2.TLSClientConfig = t.TLSClientConfig.Clone() + } if !t.tlsNextProtoWasNil { npm := map[string]func(authority string, c *tls.Conn) RoundTripper{} for k, v := range t.TLSNextProto { @@ -322,6 +334,10 @@ type h2Transport interface { CloseIdleConnections() } +func (t *Transport) hasCustomTLSDialer() bool { + return t.DialTLS != nil || t.DialTLSContext != nil +} + // onceSetNextProtoDefaults initializes TLSNextProto. // It must be called via t.nextProtoOnce.Do. func (t *Transport) onceSetNextProtoDefaults() { @@ -350,7 +366,7 @@ func (t *Transport) onceSetNextProtoDefaults() { // Transport. return } - if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil || t.DialContext != nil) { + if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()) { // Be conservative and don't automatically enable // http2 if they've specified a custom TLS config or // custom dialers. Let them opt-in themselves via @@ -359,6 +375,9 @@ func (t *Transport) onceSetNextProtoDefaults() { // However, if ForceAttemptHTTP2 is true, it overrides the above checks. return } + if omitBundledHTTP2 { + return + } t2, err := http2configureTransport(t) if err != nil { log.Printf("Error enabling Transport HTTP/2 support: %v", err) @@ -437,7 +456,7 @@ func (tr *transportRequest) setError(err error) { tr.mu.Unlock() } -// useRegisteredProtocol reports whether an alternate protocol (as reqistered +// useRegisteredProtocol reports whether an alternate protocol (as registered // with Transport.RegisterProtocol) should be respected for this request. func (t *Transport) useRegisteredProtocol(req *Request) bool { if req.URL.Scheme == "https" && req.requiresHTTP1() { @@ -469,10 +488,12 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { if isHTTP { for k, vv := range req.Header { if !httpguts.ValidHeaderFieldName(k) { + req.closeBody() return nil, fmt.Errorf("net/http: invalid header field name %q", k) } for _, v := range vv { if !httpguts.ValidHeaderFieldValue(v) { + req.closeBody() return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k) } } @@ -492,6 +513,7 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { return nil, &badStringError{"unsupported protocol scheme", scheme} } if req.Method != "" && !validMethod(req.Method) { + req.closeBody() return nil, fmt.Errorf("net/http: invalid method %q", req.Method) } if req.URL.Host == "" { @@ -537,9 +559,16 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { if err == nil { return resp, nil } - if http2isNoCachedConnError(err) { - t.removeIdleConn(pconn) - } else if !pconn.shouldRetryRequest(req, err) { + + // Failed. Clean up and determine whether to retry. + + _, isH2DialError := pconn.alt.(http2erringRoundTripper) + if http2isNoCachedConnError(err) || isH2DialError { + if t.removeIdleConn(pconn) { + t.decConnsPerHost(pconn.cacheKey) + } + } + if !pconn.shouldRetryRequest(req, err) { // Issue 16465: return underlying net.Conn.Read error from peek, // as we've historically done. if e, ok := err.(transportReadFromServerError); ok { @@ -710,20 +739,10 @@ func resetProxyConfig() { } func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { - // TODO: the validPort check is redundant after CL 189258, as url.URL.Port - // only returns valid ports now. golang.org/issue/33600 - if port := treq.URL.Port(); !validPort(port) { - return cm, fmt.Errorf("invalid URL port %q", port) - } cm.targetScheme = treq.URL.Scheme cm.targetAddr = canonicalAddr(treq.URL) if t.Proxy != nil { cm.proxyURL, err = t.Proxy(treq.Request) - if err == nil && cm.proxyURL != nil { - if port := cm.proxyURL.Port(); !validPort(port) { - return cm, fmt.Errorf("invalid proxy URL port %q", port) - } - } } cm.onlyH1 = treq.requiresHTTP1() return cm, err @@ -753,7 +772,6 @@ var ( errCloseIdleConns = errors.New("http: CloseIdleConnections called") errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") errIdleConnTimeout = errors.New("http: idle connection timeout") - errNotCachingH2Conn = errors.New("http: not caching alternate protocol's connections") // errServerClosedIdle is not seen by users for idempotent requests, but may be // seen by a user if the server shuts down an idle connection and sends its FIN @@ -911,16 +929,37 @@ func (t *Transport) queueForIdleConn(w *wantConn) (delivered bool) { return false } + // If IdleConnTimeout is set, calculate the oldest + // persistConn.idleAt time we're willing to use a cached idle + // conn. + var oldTime time.Time + if t.IdleConnTimeout > 0 { + oldTime = time.Now().Add(-t.IdleConnTimeout) + } + // Look for most recently-used idle connection. if list, ok := t.idleConn[w.key]; ok { stop := false delivered := false for len(list) > 0 && !stop { pconn := list[len(list)-1] - if pconn.isBroken() { - // persistConn.readLoop has marked the connection broken, - // but Transport.removeIdleConn has not yet removed it from the idle list. - // Drop on floor on behalf of Transport.removeIdleConn. + + // See whether this connection has been idle too long, considering + // only the wall time (the Round(0)), in case this is a laptop or VM + // coming out of suspend with previously cached idle connections. + tooOld := !oldTime.IsZero() && pconn.idleAt.Round(0).Before(oldTime) + if tooOld { + // Async cleanup. Launch in its own goroutine (as if a + // time.AfterFunc called it); it acquires idleMu, which we're + // holding, and does a synchronous net.Conn.Close. + go pconn.closeConnIfStillIdle() + } + if pconn.isBroken() || tooOld { + // If either persistConn.readLoop has marked the connection + // broken, but Transport.removeIdleConn has not yet removed it + // from the idle list, or if this persistConn is too old (it was + // idle too long), then ignore it and look for another. In both + // cases it's already in the process of being closed. list = list[:len(list)-1] continue } @@ -960,26 +999,28 @@ func (t *Transport) queueForIdleConn(w *wantConn) (delivered bool) { } // removeIdleConn marks pconn as dead. -func (t *Transport) removeIdleConn(pconn *persistConn) { +func (t *Transport) removeIdleConn(pconn *persistConn) bool { t.idleMu.Lock() defer t.idleMu.Unlock() - t.removeIdleConnLocked(pconn) + return t.removeIdleConnLocked(pconn) } // t.idleMu must be held. -func (t *Transport) removeIdleConnLocked(pconn *persistConn) { +func (t *Transport) removeIdleConnLocked(pconn *persistConn) bool { if pconn.idleTimer != nil { pconn.idleTimer.Stop() } t.idleLRU.remove(pconn) key := pconn.cacheKey pconns := t.idleConn[key] + var removed bool switch len(pconns) { case 0: // Nothing case 1: if pconns[0] == pconn { delete(t.idleConn, key) + removed = true } default: for i, v := range pconns { @@ -990,9 +1031,11 @@ func (t *Transport) removeIdleConnLocked(pconn *persistConn) { // conns at the end. copy(pconns[i:], pconns[i+1:]) t.idleConn[key] = pconns[:len(pconns)-1] + removed = true break } } + return removed } func (t *Transport) setReqCanceler(r *Request, fn func(error)) { @@ -1177,6 +1220,18 @@ func (q *wantConnQueue) cleanFront() (cleaned bool) { } } +func (t *Transport) customDialTLS(ctx context.Context, network, addr string) (conn net.Conn, err error) { + if t.DialTLSContext != nil { + conn, err = t.DialTLSContext(ctx, network, addr) + } else { + conn, err = t.DialTLS(network, addr) + } + if conn == nil && err == nil { + err = errors.New("net/http: Transport.DialTLS or DialTLSContext returned (nil, nil)") + } + return +} + // getConn dials and creates a new persistConn to the target as // specified in the connectMethod. This includes doing a proxy CONNECT // and/or setting up TLS. If this doesn't return an error, the persistConn @@ -1206,7 +1261,9 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (pc *persi // Queue for idle connection. if delivered := t.queueForIdleConn(w); delivered { pc := w.pc - if trace != nil && trace.GotConn != nil { + // Trace only for HTTP/1. + // HTTP/2 calls trace.GotConn itself. + if pc.alt == nil && trace != nil && trace.GotConn != nil { trace.GotConn(pc.gotIdleConnTrace(pc.idleAt)) } // set request canceler to some non-nil function so we @@ -1360,19 +1417,6 @@ func (t *Transport) decConnsPerHost(key connectMethodKey) { } } -// The connect method and the transport can both specify a TLS -// Host name. The transport's name takes precedence if present. -func chooseTLSHost(cm connectMethod, t *Transport) string { - tlsHost := "" - if t.TLSClientConfig != nil { - tlsHost = t.TLSClientConfig.ServerName - } - if tlsHost == "" { - tlsHost = cm.tlsHost() - } - return tlsHost -} - // Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS // tunnel, this function establishes a nested TLS session inside the encrypted channel. // The remote endpoint's name may be overridden by TLSClientConfig.ServerName. @@ -1438,15 +1482,12 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *pers } return err } - if cm.scheme() == "https" && t.DialTLS != nil { + if cm.scheme() == "https" && t.hasCustomTLSDialer() { var err error - pconn.conn, err = t.DialTLS("tcp", cm.addr()) + pconn.conn, err = t.customDialTLS(ctx, "tcp", cm.addr()) if err != nil { return nil, wrapErr(err) } - if pconn.conn == nil { - return nil, wrapErr(errors.New("net/http: Transport.DialTLS returned (nil, nil)")) - } if tc, ok := pconn.conn.(*tls.Conn); ok { // Handshake here, in case DialTLS didn't. TLSNextProto below // depends on it for knowing the connection state. @@ -1527,13 +1568,44 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *pers if pa := cm.proxyAuth(); pa != "" { connectReq.Header.Set("Proxy-Authorization", pa) } - connectReq.Write(conn) - // Read response. - // Okay to use and discard buffered reader here, because - // TLS server will not speak until spoken to. - br := bufio.NewReader(conn) - resp, err := ReadResponse(br, connectReq) + // If there's no done channel (no deadline or cancellation + // from the caller possible), at least set some (long) + // timeout here. This will make sure we don't block forever + // and leak a goroutine if the connection stops replying + // after the TCP connect. + connectCtx := ctx + if ctx.Done() == nil { + newCtx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + connectCtx = newCtx + } + + didReadResponse := make(chan struct{}) // closed after CONNECT write+read is done or fails + var ( + resp *Response + err error // write or read error + ) + // Write the CONNECT request & read the response. + go func() { + defer close(didReadResponse) + err = connectReq.Write(conn) + if err != nil { + return + } + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(conn) + resp, err = ReadResponse(br, connectReq) + }() + select { + case <-connectCtx.Done(): + conn.Close() + <-didReadResponse + return nil, connectCtx.Err() + case <-didReadResponse: + // resp or err now set + } if err != nil { conn.Close() return nil, err @@ -1927,7 +1999,7 @@ func (pc *persistConn) readLoop() { } return } - pc.readLimit = maxInt64 // effictively no limit for response bodies + pc.readLimit = maxInt64 // effectively no limit for response bodies pc.mu.Lock() pc.numExpectedResponses-- @@ -2635,11 +2707,6 @@ func (gz *gzipReader) Close() error { return gz.body.Close() } -type readerAndCloser struct { - io.Reader - io.Closer -} - type tlsHandshakeTimeoutError struct{} func (tlsHandshakeTimeoutError) Timeout() bool { return true } @@ -2702,15 +2769,3 @@ func (cl *connLRU) remove(pc *persistConn) { func (cl *connLRU) len() int { return len(cl.m) } - -// validPort reports whether p (without the colon) is a valid port in -// a URL, per RFC 3986 Section 3.2.3, which says the port may be -// empty, or only contain digits. -func validPort(p string) bool { - for _, r := range []byte(p) { - if r < '0' || r > '9' { - return false - } - } - return true -} diff --git a/libgo/go/net/http/transport_test.go b/libgo/go/net/http/transport_test.go index f304a7b..2256813 100644 --- a/libgo/go/net/http/transport_test.go +++ b/libgo/go/net/http/transport_test.go @@ -591,6 +591,7 @@ func TestTransportMaxConnsPerHostIncludeDialInProgress(t *testing.T) { func TestTransportMaxConnsPerHost(t *testing.T) { defer afterTest(t) + CondSkipHTTP2(t) h := HandlerFunc(func(w ResponseWriter, r *Request) { _, err := w.Write([]byte("foo")) @@ -1441,6 +1442,72 @@ func TestTransportProxy(t *testing.T) { } } +// Issue 28012: verify that the Transport closes its TCP connection to http proxies +// when they're slow to reply to HTTPS CONNECT responses. +func TestTransportProxyHTTPSConnectLeak(t *testing.T) { + setParallel(t) + defer afterTest(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ln := newLocalListener(t) + defer ln.Close() + listenerDone := make(chan struct{}) + go func() { + defer close(listenerDone) + c, err := ln.Accept() + if err != nil { + t.Errorf("Accept: %v", err) + return + } + defer c.Close() + // Read the CONNECT request + br := bufio.NewReader(c) + cr, err := ReadRequest(br) + if err != nil { + t.Errorf("proxy server failed to read CONNECT request") + return + } + if cr.Method != "CONNECT" { + t.Errorf("unexpected method %q", cr.Method) + return + } + + // Now hang and never write a response; instead, cancel the request and wait + // for the client to close. + // (Prior to Issue 28012 being fixed, we never closed.) + cancel() + var buf [1]byte + _, err = br.Read(buf[:]) + if err != io.EOF { + t.Errorf("proxy server Read err = %v; want EOF", err) + } + return + }() + + c := &Client{ + Transport: &Transport{ + Proxy: func(*Request) (*url.URL, error) { + return url.Parse("http://" + ln.Addr().String()) + }, + }, + } + req, err := NewRequestWithContext(ctx, "GET", "https://golang.fake.tld/", nil) + if err != nil { + t.Fatal(err) + } + _, err = c.Do(req) + if err == nil { + t.Errorf("unexpected Get success") + } + + // Wait unconditionally for the listener goroutine to exit: this should never + // hang, so if it does we want a full goroutine dump — and that's exactly what + // the testing package will give us when the test run times out. + <-listenerDone +} + // Issue 16997: test transport dial preserves typed errors func TestTransportDialPreservesNetOpProxyError(t *testing.T) { defer afterTest(t) @@ -2293,7 +2360,7 @@ func TestTransportCancelRequestInDial(t *testing.T) { got := logbuf.String() want := `dial: blocking canceling -Get = Get http://something.no-network.tld/: net/http: request canceled while waiting for connection +Get = Get "http://something.no-network.tld/": net/http: request canceled while waiting for connection ` if got != want { t.Errorf("Got events:\n%s\nWant:\n%s", got, want) @@ -3509,6 +3576,90 @@ func TestTransportDialTLS(t *testing.T) { } } +func TestTransportDialContext(t *testing.T) { + setParallel(t) + defer afterTest(t) + var mu sync.Mutex // guards following + var gotReq bool + var receivedContext context.Context + + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + mu.Lock() + gotReq = true + mu.Unlock() + })) + defer ts.Close() + c := ts.Client() + c.Transport.(*Transport).DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) { + mu.Lock() + receivedContext = ctx + mu.Unlock() + return net.Dial(netw, addr) + } + + req, err := NewRequest("GET", ts.URL, nil) + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), "some-key", "some-value") + res, err := c.Do(req.WithContext(ctx)) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + mu.Lock() + if !gotReq { + t.Error("didn't get request") + } + if receivedContext != ctx { + t.Error("didn't receive correct context") + } +} + +func TestTransportDialTLSContext(t *testing.T) { + setParallel(t) + defer afterTest(t) + var mu sync.Mutex // guards following + var gotReq bool + var receivedContext context.Context + + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { + mu.Lock() + gotReq = true + mu.Unlock() + })) + defer ts.Close() + c := ts.Client() + c.Transport.(*Transport).DialTLSContext = func(ctx context.Context, netw, addr string) (net.Conn, error) { + mu.Lock() + receivedContext = ctx + mu.Unlock() + c, err := tls.Dial(netw, addr, c.Transport.(*Transport).TLSClientConfig) + if err != nil { + return nil, err + } + return c, c.Handshake() + } + + req, err := NewRequest("GET", ts.URL, nil) + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), "some-key", "some-value") + res, err := c.Do(req.WithContext(ctx)) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + mu.Lock() + if !gotReq { + t.Error("didn't get request") + } + if receivedContext != ctx { + t.Error("didn't receive correct context") + } +} + // Test for issue 8755 // Ensure that if a proxy returns an error, it is exposed by RoundTrip func TestRoundTripReturnsProxyError(t *testing.T) { @@ -3566,7 +3717,77 @@ func TestTransportCloseIdleConnsThenReturn(t *testing.T) { wantIdle("after final put", 1) } -// This tests that an client requesting a content range won't also +// Test for issue 34282 +// Ensure that getConn doesn't call the GotConn trace hook on a HTTP/2 idle conn +func TestTransportTraceGotConnH2IdleConns(t *testing.T) { + tr := &Transport{} + wantIdle := func(when string, n int) bool { + got := tr.IdleConnCountForTesting("https", "example.com:443") // key used by PutIdleTestConnH2 + if got == n { + return true + } + t.Errorf("%s: idle conns = %d; want %d", when, got, n) + return false + } + wantIdle("start", 0) + alt := funcRoundTripper(func() {}) + if !tr.PutIdleTestConnH2("https", "example.com:443", alt) { + t.Fatal("put failed") + } + wantIdle("after put", 1) + ctx := httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ + GotConn: func(httptrace.GotConnInfo) { + // tr.getConn should leave it for the HTTP/2 alt to call GotConn. + t.Error("GotConn called") + }, + }) + req, _ := NewRequestWithContext(ctx, MethodGet, "https://example.com", nil) + _, err := tr.RoundTrip(req) + if err != errFakeRoundTrip { + t.Errorf("got error: %v; want %q", err, errFakeRoundTrip) + } + wantIdle("after round trip", 1) +} + +func TestTransportRemovesH2ConnsAfterIdle(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + trFunc := func(tr *Transport) { + tr.MaxConnsPerHost = 1 + tr.MaxIdleConnsPerHost = 1 + tr.IdleConnTimeout = 10 * time.Millisecond + } + cst := newClientServerTest(t, h2Mode, HandlerFunc(func(w ResponseWriter, r *Request) {}), trFunc) + defer cst.close() + + if _, err := cst.c.Get(cst.ts.URL); err != nil { + t.Fatalf("got error: %s", err) + } + + time.Sleep(100 * time.Millisecond) + got := make(chan error) + go func() { + if _, err := cst.c.Get(cst.ts.URL); err != nil { + got <- err + } + close(got) + }() + + timeout := time.NewTimer(5 * time.Second) + defer timeout.Stop() + select { + case err := <-got: + if err != nil { + t.Fatalf("got error: %s", err) + } + case <-timeout.C: + t.Fatal("request never completed") + } +} + +// This tests that a client requesting a content range won't also // implicitly ask for gzip support. If they want that, they need to do it // on their own. // golang.org/issue/8923 @@ -3928,6 +4149,7 @@ func TestTransportAutomaticHTTP2_DialTLS(t *testing.T) { } func testTransportAutoHTTP(t *testing.T, tr *Transport, wantH2 bool) { + CondSkipHTTP2(t) _, err := tr.RoundTrip(new(Request)) if err == nil { t.Error("expected error from RoundTrip") @@ -5509,6 +5731,7 @@ func TestTransportClone(t *testing.T) { DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { panic("") }, Dial: func(network, addr string) (net.Conn, error) { panic("") }, DialTLS: func(network, addr string) (net.Conn, error) { panic("") }, + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { panic("") }, TLSClientConfig: new(tls.Config), TLSHandshakeTimeout: time.Second, DisableKeepAlives: true, @@ -5626,3 +5849,263 @@ func TestTransportIgnores408(t *testing.T) { } t.Fatalf("timeout after %v waiting for Transport connections to die off", time.Since(t0)) } + +func TestInvalidHeaderResponse(t *testing.T) { + setParallel(t) + defer afterTest(t) + cst := newClientServerTest(t, h1Mode, HandlerFunc(func(w ResponseWriter, r *Request) { + conn, buf, _ := w.(Hijacker).Hijack() + buf.Write([]byte("HTTP/1.1 200 OK\r\n" + + "Date: Wed, 30 Aug 2017 19:09:27 GMT\r\n" + + "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Length: 0\r\n" + + "Foo : bar\r\n\r\n")) + buf.Flush() + conn.Close() + })) + defer cst.close() + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if v := res.Header.Get("Foo"); v != "" { + t.Errorf(`unexpected "Foo" header: %q`, v) + } + if v := res.Header.Get("Foo "); v != "bar" { + t.Errorf(`bad "Foo " header value: %q, want %q`, v, "bar") + } +} + +type bodyCloser bool + +func (bc *bodyCloser) Close() error { + *bc = true + return nil +} +func (bc *bodyCloser) Read(b []byte) (n int, err error) { + return 0, io.EOF +} + +// Issue 35015: ensure that Transport closes the body on any error +// with an invalid request, as promised by Client.Do docs. +func TestTransportClosesBodyOnInvalidRequests(t *testing.T) { + cst := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + t.Errorf("Should not have been invoked") + })) + defer cst.Close() + + u, _ := url.Parse(cst.URL) + + tests := []struct { + name string + req *Request + wantErr string + }{ + { + name: "invalid method", + req: &Request{ + Method: " ", + URL: u, + }, + wantErr: "invalid method", + }, + { + name: "nil URL", + req: &Request{ + Method: "GET", + }, + wantErr: "nil Request.URL", + }, + { + name: "invalid header key", + req: &Request{ + Method: "GET", + Header: Header{"💡": {"emoji"}}, + URL: u, + }, + wantErr: "invalid header field name", + }, + { + name: "invalid header value", + req: &Request{ + Method: "POST", + Header: Header{"key": {"\x19"}}, + URL: u, + }, + wantErr: "invalid header field value", + }, + { + name: "non HTTP(s) scheme", + req: &Request{ + Method: "POST", + URL: &url.URL{Scheme: "faux"}, + }, + wantErr: "unsupported protocol scheme", + }, + { + name: "no Host in URL", + req: &Request{ + Method: "POST", + URL: &url.URL{Scheme: "http"}, + }, + wantErr: "no Host", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var bc bodyCloser + req := tt.req + req.Body = &bc + _, err := DefaultClient.Do(tt.req) + if err == nil { + t.Fatal("Expected an error") + } + if !bc { + t.Fatal("Expected body to have been closed") + } + if g, w := err.Error(), tt.wantErr; !strings.Contains(g, w) { + t.Fatalf("Error mismatch\n\t%q\ndoes not contain\n\t%q", g, w) + } + }) + } +} + +// breakableConn is a net.Conn wrapper with a Write method +// that will fail when its brokenState is true. +type breakableConn struct { + net.Conn + *brokenState +} + +type brokenState struct { + sync.Mutex + broken bool +} + +func (w *breakableConn) Write(b []byte) (n int, err error) { + w.Lock() + defer w.Unlock() + if w.broken { + return 0, errors.New("some write error") + } + return w.Conn.Write(b) +} + +// Issue 34978: don't cache a broken HTTP/2 connection +func TestDontCacheBrokenHTTP2Conn(t *testing.T) { + cst := newClientServerTest(t, h2Mode, HandlerFunc(func(w ResponseWriter, r *Request) {}), optQuietLog) + defer cst.close() + + var brokenState brokenState + + const numReqs = 5 + var numDials, gotConns uint32 // atomic + + cst.tr.Dial = func(netw, addr string) (net.Conn, error) { + atomic.AddUint32(&numDials, 1) + c, err := net.Dial(netw, addr) + if err != nil { + t.Errorf("unexpected Dial error: %v", err) + return nil, err + } + return &breakableConn{c, &brokenState}, err + } + + for i := 1; i <= numReqs; i++ { + brokenState.Lock() + brokenState.broken = false + brokenState.Unlock() + + // doBreak controls whether we break the TCP connection after the TLS + // handshake (before the HTTP/2 handshake). We test a few failures + // in a row followed by a final success. + doBreak := i != numReqs + + ctx := httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { + t.Logf("got conn: %v, reused=%v, wasIdle=%v, idleTime=%v", info.Conn.LocalAddr(), info.Reused, info.WasIdle, info.IdleTime) + atomic.AddUint32(&gotConns, 1) + }, + TLSHandshakeDone: func(cfg tls.ConnectionState, err error) { + brokenState.Lock() + defer brokenState.Unlock() + if doBreak { + brokenState.broken = true + } + }, + }) + req, err := NewRequestWithContext(ctx, "GET", cst.ts.URL, nil) + if err != nil { + t.Fatal(err) + } + _, err = cst.c.Do(req) + if doBreak != (err != nil) { + t.Errorf("for iteration %d, doBreak=%v; unexpected error %v", i, doBreak, err) + } + } + if got, want := atomic.LoadUint32(&gotConns), 1; int(got) != want { + t.Errorf("GotConn calls = %v; want %v", got, want) + } + if got, want := atomic.LoadUint32(&numDials), numReqs; int(got) != want { + t.Errorf("Dials = %v; want %v", got, want) + } +} + +// Issue 34941 +// When the client has too many concurrent requests on a single connection, +// http.http2noCachedConnError is reported on multiple requests. There should +// only be one decrement regardless of the number of failures. +func TestTransportDecrementConnWhenIdleConnRemoved(t *testing.T) { + defer afterTest(t) + CondSkipHTTP2(t) + + h := HandlerFunc(func(w ResponseWriter, r *Request) { + _, err := w.Write([]byte("foo")) + if err != nil { + t.Fatalf("Write: %v", err) + } + }) + + ts := httptest.NewUnstartedServer(h) + ts.EnableHTTP2 = true + ts.StartTLS() + defer ts.Close() + + c := ts.Client() + tr := c.Transport.(*Transport) + tr.MaxConnsPerHost = 1 + if err := ExportHttp2ConfigureTransport(tr); err != nil { + t.Fatalf("ExportHttp2ConfigureTransport: %v", err) + } + + errCh := make(chan error, 300) + doReq := func() { + resp, err := c.Get(ts.URL) + if err != nil { + errCh <- fmt.Errorf("request failed: %v", err) + return + } + defer resp.Body.Close() + _, err = ioutil.ReadAll(resp.Body) + if err != nil { + errCh <- fmt.Errorf("read body failed: %v", err) + } + } + + var wg sync.WaitGroup + for i := 0; i < 300; i++ { + wg.Add(1) + go func() { + defer wg.Done() + doReq() + }() + } + wg.Wait() + close(errCh) + + for err := range errCh { + t.Errorf("error occurred: %v", err) + } +} diff --git a/libgo/go/net/interface.go b/libgo/go/net/interface.go index 5824856..914aaa0 100644 --- a/libgo/go/net/interface.go +++ b/libgo/go/net/interface.go @@ -10,7 +10,7 @@ import ( "time" ) -// BUG(mikio): On JS and NaCl, methods and functions related to +// BUG(mikio): On JS, methods and functions related to // Interface are not implemented. // BUG(mikio): On AIX, DragonFly BSD, NetBSD, OpenBSD, Plan 9 and diff --git a/libgo/go/net/interface_bsd_test.go b/libgo/go/net/interface_bsd_test.go new file mode 100644 index 0000000..947dde7 --- /dev/null +++ b/libgo/go/net/interface_bsd_test.go @@ -0,0 +1,60 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd openbsd + +package net + +import ( + "errors" + "fmt" + "os/exec" + "runtime" +) + +func (ti *testInterface) setBroadcast(vid int) error { + if runtime.GOOS == "openbsd" { + ti.name = fmt.Sprintf("vether%d", vid) + } else { + ti.name = fmt.Sprintf("vlan%d", vid) + } + xname, err := exec.LookPath("ifconfig") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "create"}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "destroy"}, + }) + return nil +} + +func (ti *testInterface) setPointToPoint(suffix int) error { + ti.name = fmt.Sprintf("gif%d", suffix) + xname, err := exec.LookPath("ifconfig") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "create"}, + }) + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "inet", ti.local, ti.remote}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "destroy"}, + }) + return nil +} + +func (ti *testInterface) setLinkLocal(suffix int) error { + return errors.New("not yet implemented for BSD") +} diff --git a/libgo/go/net/interface_linux_test.go b/libgo/go/net/interface_linux_test.go new file mode 100644 index 0000000..0699fec --- /dev/null +++ b/libgo/go/net/interface_linux_test.go @@ -0,0 +1,133 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "fmt" + "os/exec" + "testing" +) + +func (ti *testInterface) setBroadcast(suffix int) error { + ti.name = fmt.Sprintf("gotest%d", suffix) + xname, err := exec.LookPath("ip") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "link", "add", ti.name, "type", "dummy"}, + }) + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "address", "add", ti.local, "peer", ti.remote, "dev", ti.name}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "address", "del", ti.local, "peer", ti.remote, "dev", ti.name}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "link", "delete", ti.name, "type", "dummy"}, + }) + return nil +} + +func (ti *testInterface) setLinkLocal(suffix int) error { + ti.name = fmt.Sprintf("gotest%d", suffix) + xname, err := exec.LookPath("ip") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "link", "add", ti.name, "type", "dummy"}, + }) + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "address", "add", ti.local, "dev", ti.name}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "address", "del", ti.local, "dev", ti.name}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "link", "delete", ti.name, "type", "dummy"}, + }) + return nil +} + +func (ti *testInterface) setPointToPoint(suffix int) error { + ti.name = fmt.Sprintf("gotest%d", suffix) + xname, err := exec.LookPath("ip") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "tunnel", "add", ti.name, "mode", "gre", "local", ti.local, "remote", ti.remote}, + }) + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "address", "add", ti.local, "peer", ti.remote, "dev", ti.name}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "address", "del", ti.local, "peer", ti.remote, "dev", ti.name}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "tunnel", "del", ti.name, "mode", "gre", "local", ti.local, "remote", ti.remote}, + }) + return nil +} + +const ( + numOfTestIPv4MCAddrs = 14 + numOfTestIPv6MCAddrs = 18 +) + +var ( + igmpInterfaceTable = []Interface{ + {Name: "lo"}, + {Name: "eth0"}, {Name: "eth1"}, {Name: "eth2"}, + {Name: "eth0.100"}, {Name: "eth0.101"}, {Name: "eth0.102"}, {Name: "eth0.103"}, + {Name: "device1tap2"}, + } + igmp6InterfaceTable = []Interface{ + {Name: "lo"}, + {Name: "eth0"}, {Name: "eth1"}, {Name: "eth2"}, + {Name: "eth0.100"}, {Name: "eth0.101"}, {Name: "eth0.102"}, {Name: "eth0.103"}, + {Name: "device1tap2"}, + {Name: "pan0"}, + } +) + +func TestParseProcNet(t *testing.T) { + defer func() { + if p := recover(); p != nil { + t.Fatalf("panicked: %v", p) + } + }() + + var ifmat4 []Addr + for _, ifi := range igmpInterfaceTable { + ifmat := parseProcNetIGMP("testdata/igmp", &ifi) + ifmat4 = append(ifmat4, ifmat...) + } + if len(ifmat4) != numOfTestIPv4MCAddrs { + t.Fatalf("got %d; want %d", len(ifmat4), numOfTestIPv4MCAddrs) + } + + var ifmat6 []Addr + for _, ifi := range igmp6InterfaceTable { + ifmat := parseProcNetIGMP6("testdata/igmp6", &ifi) + ifmat6 = append(ifmat6, ifmat...) + } + if len(ifmat6) != numOfTestIPv6MCAddrs { + t.Fatalf("got %d; want %d", len(ifmat6), numOfTestIPv6MCAddrs) + } +} diff --git a/libgo/go/net/interface_plan9.go b/libgo/go/net/interface_plan9.go index 8fe9138..1295017 100644 --- a/libgo/go/net/interface_plan9.go +++ b/libgo/go/net/interface_plan9.go @@ -143,8 +143,8 @@ func interfaceAddrTable(ifi *Interface) ([]Addr, error) { ifcs = []Interface{*ifi} } - addrs := make([]Addr, len(ifcs)) - for i, ifc := range ifcs { + var addrs []Addr + for _, ifc := range ifcs { status := ifc.Name + "/status" statusf, err := open(status) if err != nil { @@ -157,39 +157,36 @@ func interfaceAddrTable(ifi *Interface) ([]Addr, error) { if _, ok := statusf.readLine(); !ok { return nil, errors.New("cannot read header line for interface: " + status) } - line, ok := statusf.readLine() - if !ok { - return nil, errors.New("cannot read IP address for interface: " + status) - } - // This assumes only a single address for the interface. - fields := getFields(line) - if len(fields) < 1 { - return nil, errors.New("cannot parse IP address for interface: " + status) - } - addr := fields[0] - ip := ParseIP(addr) - if ip == nil { - return nil, errors.New("cannot parse IP address for interface: " + status) - } + for line, ok := statusf.readLine(); ok; line, ok = statusf.readLine() { + fields := getFields(line) + if len(fields) < 1 { + return nil, errors.New("cannot parse IP address for interface: " + status) + } + addr := fields[0] + ip := ParseIP(addr) + if ip == nil { + return nil, errors.New("cannot parse IP address for interface: " + status) + } - // The mask is represented as CIDR relative to the IPv6 address. - // Plan 9 internal representation is always IPv6. - maskfld := fields[1] - maskfld = maskfld[1:] - pfxlen, _, ok := dtoi(maskfld) - if !ok { - return nil, errors.New("cannot parse network mask for interface: " + status) - } - var mask IPMask - if ip.To4() != nil { // IPv4 or IPv6 IPv4-mapped address - mask = CIDRMask(pfxlen-8*len(v4InV6Prefix), 8*IPv4len) - } - if ip.To16() != nil && ip.To4() == nil { // IPv6 address - mask = CIDRMask(pfxlen, 8*IPv6len) - } + // The mask is represented as CIDR relative to the IPv6 address. + // Plan 9 internal representation is always IPv6. + maskfld := fields[1] + maskfld = maskfld[1:] + pfxlen, _, ok := dtoi(maskfld) + if !ok { + return nil, errors.New("cannot parse network mask for interface: " + status) + } + var mask IPMask + if ip.To4() != nil { // IPv4 or IPv6 IPv4-mapped address + mask = CIDRMask(pfxlen-8*len(v4InV6Prefix), 8*IPv4len) + } + if ip.To16() != nil && ip.To4() == nil { // IPv6 address + mask = CIDRMask(pfxlen, 8*IPv6len) + } - addrs[i] = &IPNet{IP: ip, Mask: mask} + addrs = append(addrs, &IPNet{IP: ip, Mask: mask}) + } } return addrs, nil diff --git a/libgo/go/net/interface_stub.go b/libgo/go/net/interface_stub.go index d8afd5e..437b9eb 100644 --- a/libgo/go/net/interface_stub.go +++ b/libgo/go/net/interface_stub.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build nacl hurd js,wasm +// +build hurd js,wasm package net diff --git a/libgo/go/net/interface_test.go b/libgo/go/net/interface_test.go index fb6032f..b2ef21e 100644 --- a/libgo/go/net/interface_test.go +++ b/libgo/go/net/interface_test.go @@ -278,7 +278,7 @@ func checkUnicastStats(ifStats *ifStats, uniStats *routeStats) error { func checkMulticastStats(ifStats *ifStats, uniStats, multiStats *routeStats) error { switch runtime.GOOS { - case "aix", "dragonfly", "nacl", "netbsd", "openbsd", "plan9", "solaris", "illumos": + case "aix", "dragonfly", "netbsd", "openbsd", "plan9", "solaris", "illumos": default: // Test the existence of connected multicast route // clones for IPv4. Unlike IPv6, IPv4 multicast diff --git a/libgo/go/net/interface_unix_test.go b/libgo/go/net/interface_unix_test.go new file mode 100644 index 0000000..6a2b7f1 --- /dev/null +++ b/libgo/go/net/interface_unix_test.go @@ -0,0 +1,212 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd + +package net + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "testing" + "time" +) + +type testInterface struct { + name string + local string + remote string + setupCmds []*exec.Cmd + teardownCmds []*exec.Cmd +} + +func (ti *testInterface) setup() error { + for _, cmd := range ti.setupCmds { + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("args=%v out=%q err=%v", cmd.Args, string(out), err) + } + } + return nil +} + +func (ti *testInterface) teardown() error { + for _, cmd := range ti.teardownCmds { + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("args=%v out=%q err=%v ", cmd.Args, string(out), err) + } + } + return nil +} + +func TestPointToPointInterface(t *testing.T) { + if testing.Short() { + t.Skip("avoid external network") + } + if runtime.GOOS == "darwin" { + t.Skipf("not supported on %s", runtime.GOOS) + } + if os.Getuid() != 0 { + t.Skip("must be root") + } + + // We suppose that using IPv4 link-local addresses doesn't + // harm anyone. + local, remote := "169.254.0.1", "169.254.0.254" + ip := ParseIP(remote) + for i := 0; i < 3; i++ { + ti := &testInterface{local: local, remote: remote} + if err := ti.setPointToPoint(5963 + i); err != nil { + t.Skipf("test requires external command: %v", err) + } + if err := ti.setup(); err != nil { + if e := err.Error(); strings.Contains(e, "No such device") && strings.Contains(e, "gre0") { + t.Skip("skipping test; no gre0 device. likely running in container?") + } + t.Fatal(err) + } else { + time.Sleep(3 * time.Millisecond) + } + ift, err := Interfaces() + if err != nil { + ti.teardown() + t.Fatal(err) + } + for _, ifi := range ift { + if ti.name != ifi.Name { + continue + } + ifat, err := ifi.Addrs() + if err != nil { + ti.teardown() + t.Fatal(err) + } + for _, ifa := range ifat { + if ip.Equal(ifa.(*IPNet).IP) { + ti.teardown() + t.Fatalf("got %v", ifa) + } + } + } + if err := ti.teardown(); err != nil { + t.Fatal(err) + } else { + time.Sleep(3 * time.Millisecond) + } + } +} + +func TestInterfaceArrivalAndDeparture(t *testing.T) { + if testing.Short() { + t.Skip("avoid external network") + } + if os.Getuid() != 0 { + t.Skip("must be root") + } + + // We suppose that using IPv4 link-local addresses and the + // dot1Q ID for Token Ring and FDDI doesn't harm anyone. + local, remote := "169.254.0.1", "169.254.0.254" + ip := ParseIP(remote) + for _, vid := range []int{1002, 1003, 1004, 1005} { + ift1, err := Interfaces() + if err != nil { + t.Fatal(err) + } + ti := &testInterface{local: local, remote: remote} + if err := ti.setBroadcast(vid); err != nil { + t.Skipf("test requires external command: %v", err) + } + if err := ti.setup(); err != nil { + t.Fatal(err) + } else { + time.Sleep(3 * time.Millisecond) + } + ift2, err := Interfaces() + if err != nil { + ti.teardown() + t.Fatal(err) + } + if len(ift2) <= len(ift1) { + for _, ifi := range ift1 { + t.Logf("before: %v", ifi) + } + for _, ifi := range ift2 { + t.Logf("after: %v", ifi) + } + ti.teardown() + t.Fatalf("got %v; want gt %v", len(ift2), len(ift1)) + } + for _, ifi := range ift2 { + if ti.name != ifi.Name { + continue + } + ifat, err := ifi.Addrs() + if err != nil { + ti.teardown() + t.Fatal(err) + } + for _, ifa := range ifat { + if ip.Equal(ifa.(*IPNet).IP) { + ti.teardown() + t.Fatalf("got %v", ifa) + } + } + } + if err := ti.teardown(); err != nil { + t.Fatal(err) + } else { + time.Sleep(3 * time.Millisecond) + } + ift3, err := Interfaces() + if err != nil { + t.Fatal(err) + } + if len(ift3) >= len(ift2) { + for _, ifi := range ift2 { + t.Logf("before: %v", ifi) + } + for _, ifi := range ift3 { + t.Logf("after: %v", ifi) + } + t.Fatalf("got %v; want lt %v", len(ift3), len(ift2)) + } + } +} + +func TestInterfaceArrivalAndDepartureZoneCache(t *testing.T) { + if testing.Short() { + t.Skip("avoid external network") + } + if os.Getuid() != 0 { + t.Skip("must be root") + } + + // Ensure zoneCache is filled: + _, _ = Listen("tcp", "[fe80::1%nonexistent]:0") + + ti := &testInterface{local: "fe80::1"} + if err := ti.setLinkLocal(0); err != nil { + t.Skipf("test requires external command: %v", err) + } + if err := ti.setup(); err != nil { + t.Fatal(err) + } + defer ti.teardown() + + time.Sleep(3 * time.Millisecond) + + // If Listen fails (on Linux with “bind: invalid argument”), zoneCache was + // not updated when encountering a nonexistent interface: + ln, err := Listen("tcp", "[fe80::1%"+ti.name+"]:0") + if err != nil { + t.Fatal(err) + } + ln.Close() + if err := ti.teardown(); err != nil { + t.Fatal(err) + } +} diff --git a/libgo/go/net/interface_windows.go b/libgo/go/net/interface_windows.go index 28b0a65..5449432 100644 --- a/libgo/go/net/interface_windows.go +++ b/libgo/go/net/interface_windows.go @@ -58,7 +58,7 @@ func interfaceTable(ifindex int) ([]Interface, error) { if ifindex == 0 || ifindex == int(index) { ifi := Interface{ Index: int(index), - Name: syscall.UTF16ToString((*(*[10000]uint16)(unsafe.Pointer(aa.FriendlyName)))[:]), + Name: windows.UTF16PtrToString(aa.FriendlyName, 10000), } if aa.OperStatus == windows.IfOperStatusUp { ifi.Flags |= FlagUp diff --git a/libgo/go/net/internal/socktest/switch_unix.go b/libgo/go/net/internal/socktest/switch_unix.go index 2995913..4c037ba 100644 --- a/libgo/go/net/internal/socktest/switch_unix.go +++ b/libgo/go/net/internal/socktest/switch_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris package socktest diff --git a/libgo/go/net/internal/socktest/sys_unix.go b/libgo/go/net/internal/socktest/sys_unix.go index 81c7fb6..fbbffc6 100644 --- a/libgo/go/net/internal/socktest/sys_unix.go +++ b/libgo/go/net/internal/socktest/sys_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris package socktest diff --git a/libgo/go/net/ip.go b/libgo/go/net/ip.go index 1a1d0e7..9d1223e 100644 --- a/libgo/go/net/ip.go +++ b/libgo/go/net/ip.go @@ -31,7 +31,10 @@ const ( // be an IPv4 address. type IP []byte -// An IP mask is an IP address. +// An IPMask is a bitmask that can be used to manipulate +// IP addresses for IP addressing and routing. +// +// See type IPNet and func ParseCIDR for details. type IPMask []byte // An IPNet represents an IP network. @@ -65,8 +68,8 @@ func IPv4Mask(a, b, c, d byte) IPMask { return p } -// CIDRMask returns an IPMask consisting of `ones' 1 bits -// followed by 0s up to a total length of `bits' bits. +// CIDRMask returns an IPMask consisting of 'ones' 1 bits +// followed by 0s up to a total length of 'bits' bits. // For a mask of this form, CIDRMask is the inverse of IPMask.Size. func CIDRMask(ones, bits int) IPMask { if bits != 8*IPv4len && bits != 8*IPv6len { diff --git a/libgo/go/net/iprawsock.go b/libgo/go/net/iprawsock.go index 8a9c265..f18331a 100644 --- a/libgo/go/net/iprawsock.go +++ b/libgo/go/net/iprawsock.go @@ -21,7 +21,7 @@ import ( // change the behavior of these methods; use Read or ReadMsgIP // instead. -// BUG(mikio): On JS, NaCl and Plan 9, methods and functions related +// BUG(mikio): On JS and Plan 9, methods and functions related // to IPConn are not implemented. // BUG(mikio): On Windows, the File method of IPConn is not diff --git a/libgo/go/net/iprawsock_posix.go b/libgo/go/net/iprawsock_posix.go index a14c629..fdb3913 100644 --- a/libgo/go/net/iprawsock_posix.go +++ b/libgo/go/net/iprawsock_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris windows package net diff --git a/libgo/go/net/ipsock_plan9.go b/libgo/go/net/ipsock_plan9.go index d226585..93f0f4e 100644 --- a/libgo/go/net/ipsock_plan9.go +++ b/libgo/go/net/ipsock_plan9.go @@ -227,7 +227,7 @@ func listenPlan9(ctx context.Context, net string, laddr Addr) (fd *netFD, err er _, err = f.WriteString("announce " + dest) if err != nil { f.Close() - return nil, err + return nil, &OpError{Op: "announce", Net: net, Source: laddr, Addr: nil, Err: err} } laddr, err = readPlan9Addr(proto, netdir+"/"+proto+"/"+name+"/local") if err != nil { diff --git a/libgo/go/net/ipsock_posix.go b/libgo/go/net/ipsock_posix.go index 5bf5342..84e72d5 100644 --- a/libgo/go/net/ipsock_posix.go +++ b/libgo/go/net/ipsock_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris windows package net @@ -134,7 +134,7 @@ func favoriteAddrFamily(network string, laddr, raddr sockaddr, mode string) (fam } func internetSocket(ctx context.Context, net string, laddr, raddr sockaddr, sotype, proto int, mode string, ctrlFn func(string, string, syscall.RawConn) error) (fd *netFD, err error) { - if (runtime.GOOS == "aix" || runtime.GOOS == "windows" || runtime.GOOS == "openbsd" || runtime.GOOS == "nacl") && mode == "dial" && raddr.isWildcard() { + if (runtime.GOOS == "aix" || runtime.GOOS == "windows" || runtime.GOOS == "openbsd") && mode == "dial" && raddr.isWildcard() { raddr = raddr.toLocal(net) } family, ipv6only := favoriteAddrFamily(net, laddr, raddr, mode) @@ -162,7 +162,7 @@ func ipToSockaddr(family int, ip IP, port int, zone string) (syscall.Sockaddr, e // of IP node. // // When the IP node supports IPv4-mapped IPv6 address, - // we allow an listener to listen to the wildcard + // we allow a listener to listen to the wildcard // address of both IP addressing spaces by specifying // IPv6 wildcard address. if len(ip) == 0 || ip.Equal(IPv4zero) { diff --git a/libgo/go/net/listen_test.go b/libgo/go/net/listen_test.go index fef2b64..d8c7209 100644 --- a/libgo/go/net/listen_test.go +++ b/libgo/go/net/listen_test.go @@ -224,7 +224,7 @@ var dualStackTCPListenerTests = []struct { // to be greater than or equal to 4.4. func TestDualStackTCPListener(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } if !supportsIPv4() || !supportsIPv6() { @@ -314,7 +314,7 @@ var dualStackUDPListenerTests = []struct { // to be greater than or equal to 4.4. func TestDualStackUDPListener(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } if !supportsIPv4() || !supportsIPv6() { @@ -532,7 +532,7 @@ func TestIPv4MulticastListener(t *testing.T) { testenv.MustHaveExternalNetwork(t) switch runtime.GOOS { - case "android", "nacl", "plan9": + case "android", "plan9": t.Skipf("not supported on %s", runtime.GOOS) case "solaris", "illumos": t.Skipf("not supported on solaris or illumos, see golang.org/issue/7399") @@ -733,7 +733,7 @@ func TestClosingListener(t *testing.T) { func TestListenConfigControl(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } diff --git a/libgo/go/net/lookup.go b/libgo/go/net/lookup.go index 24d0d25c..9cebd10 100644 --- a/libgo/go/net/lookup.go +++ b/libgo/go/net/lookup.go @@ -27,8 +27,7 @@ var protocols = map[string]int{ } // services contains minimal mappings between services names and port -// numbers for platforms that don't have a complete list of port numbers -// (some Solaris distros, nacl, etc). +// numbers for platforms that don't have a complete list of port numbers. // // See https://www.iana.org/assignments/service-names-port-numbers // diff --git a/libgo/go/net/lookup_fake.go b/libgo/go/net/lookup_fake.go index 6c8a151..3b3c39b 100644 --- a/libgo/go/net/lookup_fake.go +++ b/libgo/go/net/lookup_fake.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build nacl js,wasm +// +build js,wasm package net diff --git a/libgo/go/net/lookup_test.go b/libgo/go/net/lookup_test.go index dd599c7..8a41510 100644 --- a/libgo/go/net/lookup_test.go +++ b/libgo/go/net/lookup_test.go @@ -21,6 +21,10 @@ import ( "time" ) +func hasSuffixFold(s, suffix string) bool { + return strings.HasSuffix(strings.ToLower(s), strings.ToLower(suffix)) +} + func lookupLocalhost(ctx context.Context, fn func(context.Context, string, string) ([]IPAddr, error), network, host string) ([]IPAddr, error) { switch host { case "localhost": @@ -97,11 +101,11 @@ func TestLookupGoogleSRV(t *testing.T) { if len(srvs) == 0 { t.Error("got no record") } - if !strings.HasSuffix(cname, tt.cname) { + if !hasSuffixFold(cname, tt.cname) { t.Errorf("got %s; want %s", cname, tt.cname) } for _, srv := range srvs { - if !strings.HasSuffix(srv.Target, tt.target) { + if !hasSuffixFold(srv.Target, tt.target) { t.Errorf("got %v; want a record containing %s", srv, tt.target) } } @@ -147,7 +151,7 @@ func TestLookupGmailMX(t *testing.T) { t.Error("got no record") } for _, mx := range mxs { - if !strings.HasSuffix(mx.Host, tt.host) { + if !hasSuffixFold(mx.Host, tt.host) { t.Errorf("got %v; want a record containing %s", mx, tt.host) } } @@ -193,7 +197,7 @@ func TestLookupGmailNS(t *testing.T) { t.Error("got no record") } for _, ns := range nss { - if !strings.HasSuffix(ns.Host, tt.host) { + if !hasSuffixFold(ns.Host, tt.host) { t.Errorf("got %v; want a record containing %s", ns, tt.host) } } @@ -279,7 +283,7 @@ func TestLookupGooglePublicDNSAddr(t *testing.T) { t.Error("got no record") } for _, name := range names { - if !strings.HasSuffix(name, ".google.com.") && !strings.HasSuffix(name, ".google.") { + if !hasSuffixFold(name, ".google.com.") && !hasSuffixFold(name, ".google.") { t.Errorf("got %q; want a record ending in .google.com. or .google.", name) } } @@ -371,7 +375,7 @@ func TestLookupCNAME(t *testing.T) { } t.Fatal(err) } - if !strings.HasSuffix(cname, tt.cname) { + if !hasSuffixFold(cname, tt.cname) { t.Errorf("got %s; want a record containing %s", cname, tt.cname) } } @@ -656,7 +660,7 @@ func testDots(t *testing.T, mode string) { t.Errorf("LookupAddr(8.8.8.8): %v (mode=%v)", err, mode) } else { for _, name := range names { - if !strings.HasSuffix(name, ".google.com.") && !strings.HasSuffix(name, ".google.") { + if !hasSuffixFold(name, ".google.com.") && !hasSuffixFold(name, ".google.") { t.Errorf("LookupAddr(8.8.8.8) = %v, want names ending in .google.com or .google with trailing dot (mode=%v)", names, mode) break } @@ -677,7 +681,7 @@ func testDots(t *testing.T, mode string) { t.Errorf("LookupMX(google.com): %v (mode=%v)", err, mode) } else { for _, mx := range mxs { - if !strings.HasSuffix(mx.Host, ".google.com.") { + if !hasSuffixFold(mx.Host, ".google.com.") { t.Errorf("LookupMX(google.com) = %v, want names ending in .google.com. with trailing dot (mode=%v)", mxString(mxs), mode) break } @@ -690,7 +694,7 @@ func testDots(t *testing.T, mode string) { t.Errorf("LookupNS(google.com): %v (mode=%v)", err, mode) } else { for _, ns := range nss { - if !strings.HasSuffix(ns.Host, ".google.com.") { + if !hasSuffixFold(ns.Host, ".google.com.") { t.Errorf("LookupNS(google.com) = %v, want names ending in .google.com. with trailing dot (mode=%v)", nsString(nss), mode) break } @@ -702,11 +706,11 @@ func testDots(t *testing.T, mode string) { testenv.SkipFlakyNet(t) t.Errorf("LookupSRV(xmpp-server, tcp, google.com): %v (mode=%v)", err, mode) } else { - if !strings.HasSuffix(cname, ".google.com.") { + if !hasSuffixFold(cname, ".google.com.") { t.Errorf("LookupSRV(xmpp-server, tcp, google.com) returned cname=%v, want name ending in .google.com. with trailing dot (mode=%v)", cname, mode) } for _, srv := range srvs { - if !strings.HasSuffix(srv.Target, ".google.com.") { + if !hasSuffixFold(srv.Target, ".google.com.") { t.Errorf("LookupSRV(xmpp-server, tcp, google.com) returned addrs=%v, want names ending in .google.com. with trailing dot (mode=%v)", srvString(srvs), mode) break } @@ -856,10 +860,6 @@ func TestLookupProtocol_Minimal(t *testing.T) { } func TestLookupNonLDH(t *testing.T) { - if runtime.GOOS == "nacl" { - t.Skip("skip on nacl") - } - defer dnsWaitGroup.Wait() if fixup := forceGoDNS(); fixup != nil { @@ -884,10 +884,6 @@ func TestLookupNonLDH(t *testing.T) { func TestLookupContextCancel(t *testing.T) { mustHaveExternalNetwork(t) - if runtime.GOOS == "nacl" { - t.Skip("skip on nacl") - } - defer dnsWaitGroup.Wait() ctx, ctxCancel := context.WithCancel(context.Background()) @@ -909,9 +905,6 @@ func TestLookupContextCancel(t *testing.T) { // crashes if nil is used. func TestNilResolverLookup(t *testing.T) { mustHaveExternalNetwork(t) - if runtime.GOOS == "nacl" { - t.Skip("skip on nacl") - } var r *Resolver = nil ctx := context.Background() @@ -931,10 +924,6 @@ func TestNilResolverLookup(t *testing.T) { // canceled lookups (see golang.org/issue/24178 for details). func TestLookupHostCancel(t *testing.T) { mustHaveExternalNetwork(t) - if runtime.GOOS == "nacl" { - t.Skip("skip on nacl") - } - const ( google = "www.google.com" invalidDomain = "invalid.invalid" // RFC 2606 reserves .invalid @@ -983,10 +972,11 @@ func (lcr *lookupCustomResolver) dial() func(ctx context.Context, network, addre // TestConcurrentPreferGoResolversDial tests that multiple resolvers with the // PreferGo option used concurrently are all dialed properly. func TestConcurrentPreferGoResolversDial(t *testing.T) { - // The windows implementation of the resolver does not use the Dial - // function. - if runtime.GOOS == "windows" { - t.Skip("skip on windows") + // The windows and plan9 implementation of the resolver does not use + // the Dial function. + switch runtime.GOOS { + case "windows", "plan9": + t.Skipf("skip on %v", runtime.GOOS) } testenv.MustHaveExternalNetwork(t) diff --git a/libgo/go/net/lookup_windows.go b/libgo/go/net/lookup_windows.go index d7b28f5..7d5c941 100644 --- a/libgo/go/net/lookup_windows.go +++ b/libgo/go/net/lookup_windows.go @@ -6,6 +6,7 @@ package net import ( "context" + "internal/syscall/windows" "os" "runtime" "syscall" @@ -233,7 +234,7 @@ func (*Resolver) lookupCNAME(ctx context.Context, name string) (string, error) { defer syscall.DnsRecordListFree(r, 1) resolved := resolveCNAME(syscall.StringToUTF16Ptr(name), r) - cname := syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(resolved))[:]) + cname := windows.UTF16PtrToString(resolved, 256) return absDomainName([]byte(cname)), nil } @@ -277,7 +278,7 @@ func (*Resolver) lookupMX(ctx context.Context, name string) ([]*MX, error) { mxs := make([]*MX, 0, 10) for _, p := range validRecs(r, syscall.DNS_TYPE_MX, name) { v := (*syscall.DNSMXData)(unsafe.Pointer(&p.Data[0])) - mxs = append(mxs, &MX{absDomainName([]byte(syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.NameExchange))[:]))), v.Preference}) + mxs = append(mxs, &MX{absDomainName([]byte(windows.UTF16PtrToString(v.NameExchange, 256))), v.Preference}) } byPref(mxs).sort() return mxs, nil @@ -317,8 +318,8 @@ func (*Resolver) lookupTXT(ctx context.Context, name string) ([]string, error) { for _, p := range validRecs(r, syscall.DNS_TYPE_TEXT, name) { d := (*syscall.DNSTXTData)(unsafe.Pointer(&p.Data[0])) s := "" - for _, v := range (*[1 << 10]*uint16)(unsafe.Pointer(&(d.StringArray[0])))[:d.StringCount] { - s += syscall.UTF16ToString((*[1 << 20]uint16)(unsafe.Pointer(v))[:]) + for _, v := range (*[1 << 10]*uint16)(unsafe.Pointer(&(d.StringArray[0])))[:d.StringCount:d.StringCount] { + s += windows.UTF16PtrToString(v, 1<<20) } txts = append(txts, s) } @@ -343,7 +344,7 @@ func (*Resolver) lookupAddr(ctx context.Context, addr string) ([]string, error) ptrs := make([]string, 0, 10) for _, p := range validRecs(r, syscall.DNS_TYPE_PTR, arpa) { v := (*syscall.DNSPTRData)(unsafe.Pointer(&p.Data[0])) - ptrs = append(ptrs, absDomainName([]byte(syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Host))[:])))) + ptrs = append(ptrs, absDomainName([]byte(windows.UTF16PtrToString(v.Host, 256)))) } return ptrs, nil } @@ -358,7 +359,8 @@ func validRecs(r *syscall.DNSRecord, dnstype uint16, name string) []*syscall.DNS } rec := make([]*syscall.DNSRecord, 0, 10) for p := r; p != nil; p = p.Next { - if p.Dw&dnsSectionMask != syscall.DnsSectionAnswer { + // in case of a local machine, DNS records are returned with DNSREC_QUESTION flag instead of DNS_ANSWER + if p.Dw&dnsSectionMask != syscall.DnsSectionAnswer && p.Dw&dnsSectionMask != syscall.DnsSectionQuestion { continue } if p.Type != dnstype { @@ -374,7 +376,7 @@ func validRecs(r *syscall.DNSRecord, dnstype uint16, name string) []*syscall.DNS // returns the last CNAME in chain func resolveCNAME(name *uint16, r *syscall.DNSRecord) *uint16 { - // limit cname resolving to 10 in case of a infinite CNAME loop + // limit cname resolving to 10 in case of an infinite CNAME loop Cname: for cnameloop := 0; cnameloop < 10; cnameloop++ { for p := r; p != nil; p = p.Next { diff --git a/libgo/go/net/lookup_windows_test.go b/libgo/go/net/lookup_windows_test.go new file mode 100644 index 0000000..62b61ed --- /dev/null +++ b/libgo/go/net/lookup_windows_test.go @@ -0,0 +1,317 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "internal/testenv" + "os/exec" + "reflect" + "regexp" + "sort" + "strings" + "testing" +) + +var nslookupTestServers = []string{"mail.golang.com", "gmail.com"} +var lookupTestIPs = []string{"8.8.8.8", "1.1.1.1"} + +func toJson(v interface{}) string { + data, _ := json.Marshal(v) + return string(data) +} + +func TestNSLookupMX(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + for _, server := range nslookupTestServers { + mx, err := LookupMX(server) + if err != nil { + t.Error(err) + continue + } + if len(mx) == 0 { + t.Errorf("no results") + continue + } + expected, err := nslookupMX(server) + if err != nil { + t.Logf("skipping failed nslookup %s test: %s", server, err) + } + sort.Sort(byPrefAndHost(expected)) + sort.Sort(byPrefAndHost(mx)) + if !reflect.DeepEqual(expected, mx) { + t.Errorf("different results %s:\texp:%v\tgot:%v", server, toJson(expected), toJson(mx)) + } + } +} + +func TestNSLookupCNAME(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + for _, server := range nslookupTestServers { + cname, err := LookupCNAME(server) + if err != nil { + t.Errorf("failed %s: %s", server, err) + continue + } + if cname == "" { + t.Errorf("no result %s", server) + } + expected, err := nslookupCNAME(server) + if err != nil { + t.Logf("skipping failed nslookup %s test: %s", server, err) + continue + } + if expected != cname { + t.Errorf("different results %s:\texp:%v\tgot:%v", server, expected, cname) + } + } +} + +func TestNSLookupNS(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + for _, server := range nslookupTestServers { + ns, err := LookupNS(server) + if err != nil { + t.Errorf("failed %s: %s", server, err) + continue + } + if len(ns) == 0 { + t.Errorf("no results") + continue + } + expected, err := nslookupNS(server) + if err != nil { + t.Logf("skipping failed nslookup %s test: %s", server, err) + continue + } + sort.Sort(byHost(expected)) + sort.Sort(byHost(ns)) + if !reflect.DeepEqual(expected, ns) { + t.Errorf("different results %s:\texp:%v\tgot:%v", toJson(server), toJson(expected), ns) + } + } +} + +func TestNSLookupTXT(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + for _, server := range nslookupTestServers { + txt, err := LookupTXT(server) + if err != nil { + t.Errorf("failed %s: %s", server, err) + continue + } + if len(txt) == 0 { + t.Errorf("no results") + continue + } + expected, err := nslookupTXT(server) + if err != nil { + t.Logf("skipping failed nslookup %s test: %s", server, err) + continue + } + sort.Strings(expected) + sort.Strings(txt) + if !reflect.DeepEqual(expected, txt) { + t.Errorf("different results %s:\texp:%v\tgot:%v", server, toJson(expected), toJson(txt)) + } + } +} + +func TestLookupLocalPTR(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + addr, err := localIP() + if err != nil { + t.Errorf("failed to get local ip: %s", err) + } + names, err := LookupAddr(addr.String()) + if err != nil { + t.Errorf("failed %s: %s", addr, err) + } + if len(names) == 0 { + t.Errorf("no results") + } + expected, err := lookupPTR(addr.String()) + if err != nil { + t.Logf("skipping failed lookup %s test: %s", addr.String(), err) + } + sort.Strings(expected) + sort.Strings(names) + if !reflect.DeepEqual(expected, names) { + t.Errorf("different results %s:\texp:%v\tgot:%v", addr, toJson(expected), toJson(names)) + } +} + +func TestLookupPTR(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + for _, addr := range lookupTestIPs { + names, err := LookupAddr(addr) + if err != nil { + t.Errorf("failed %s: %s", addr, err) + } + if len(names) == 0 { + t.Errorf("no results") + } + expected, err := lookupPTR(addr) + if err != nil { + t.Logf("skipping failed lookup %s test: %s", addr, err) + } + sort.Strings(expected) + sort.Strings(names) + if !reflect.DeepEqual(expected, names) { + t.Errorf("different results %s:\texp:%v\tgot:%v", addr, toJson(expected), toJson(names)) + } + } +} + +type byPrefAndHost []*MX + +func (s byPrefAndHost) Len() int { return len(s) } +func (s byPrefAndHost) Less(i, j int) bool { + if s[i].Pref != s[j].Pref { + return s[i].Pref < s[j].Pref + } + return s[i].Host < s[j].Host +} +func (s byPrefAndHost) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type byHost []*NS + +func (s byHost) Len() int { return len(s) } +func (s byHost) Less(i, j int) bool { return s[i].Host < s[j].Host } +func (s byHost) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func nslookup(qtype, name string) (string, error) { + var out bytes.Buffer + var err bytes.Buffer + cmd := exec.Command("nslookup", "-querytype="+qtype, name) + cmd.Stdout = &out + cmd.Stderr = &err + if err := cmd.Run(); err != nil { + return "", err + } + r := strings.ReplaceAll(out.String(), "\r\n", "\n") + // nslookup stderr output contains also debug information such as + // "Non-authoritative answer" and it doesn't return the correct errcode + if strings.Contains(err.String(), "can't find") { + return r, errors.New(err.String()) + } + return r, nil +} + +func nslookupMX(name string) (mx []*MX, err error) { + var r string + if r, err = nslookup("mx", name); err != nil { + return + } + mx = make([]*MX, 0, 10) + // linux nslookup syntax + // golang.org mail exchanger = 2 alt1.aspmx.l.google.com. + rx := regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+mail exchanger\s*=\s*([0-9]+)\s*([a-z0-9.\-]+)$`) + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + pref, _, _ := dtoi(ans[2]) + mx = append(mx, &MX{absDomainName([]byte(ans[3])), uint16(pref)}) + } + // windows nslookup syntax + // gmail.com MX preference = 30, mail exchanger = alt3.gmail-smtp-in.l.google.com + rx = regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+MX preference\s*=\s*([0-9]+)\s*,\s*mail exchanger\s*=\s*([a-z0-9.\-]+)$`) + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + pref, _, _ := dtoi(ans[2]) + mx = append(mx, &MX{absDomainName([]byte(ans[3])), uint16(pref)}) + } + return +} + +func nslookupNS(name string) (ns []*NS, err error) { + var r string + if r, err = nslookup("ns", name); err != nil { + return + } + ns = make([]*NS, 0, 10) + // golang.org nameserver = ns1.google.com. + rx := regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+nameserver\s*=\s*([a-z0-9.\-]+)$`) + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + ns = append(ns, &NS{absDomainName([]byte(ans[2]))}) + } + return +} + +func nslookupCNAME(name string) (cname string, err error) { + var r string + if r, err = nslookup("cname", name); err != nil { + return + } + // mail.golang.com canonical name = golang.org. + rx := regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+canonical name\s*=\s*([a-z0-9.\-]+)$`) + // assumes the last CNAME is the correct one + last := name + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + last = ans[2] + } + return absDomainName([]byte(last)), nil +} + +func nslookupTXT(name string) (txt []string, err error) { + var r string + if r, err = nslookup("txt", name); err != nil { + return + } + txt = make([]string, 0, 10) + // linux + // golang.org text = "v=spf1 redirect=_spf.google.com" + + // windows + // golang.org text = + // + // "v=spf1 redirect=_spf.google.com" + rx := regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+text\s*=\s*"(.*)"$`) + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + txt = append(txt, ans[2]) + } + return +} + +func ping(name string) (string, error) { + cmd := exec.Command("ping", "-n", "1", "-a", name) + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("%v: %v", err, string(stdoutStderr)) + } + r := strings.ReplaceAll(string(stdoutStderr), "\r\n", "\n") + return r, nil +} + +func lookupPTR(name string) (ptr []string, err error) { + var r string + if r, err = ping(name); err != nil { + return + } + ptr = make([]string, 0, 10) + rx := regexp.MustCompile(`(?m)^Pinging\s+([a-zA-Z0-9.\-]+)\s+\[.*$`) + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + ptr = append(ptr, ans[1]+".") + } + return +} + +func localIP() (ip IP, err error) { + conn, err := Dial("udp", "golang.org:80") + if err != nil { + return nil, err + } + defer conn.Close() + + localAddr := conn.LocalAddr().(*UDPAddr) + + return localAddr.IP, nil +} diff --git a/libgo/go/net/mail/message.go b/libgo/go/net/mail/message.go index 75207db..0781310 100644 --- a/libgo/go/net/mail/message.go +++ b/libgo/go/net/mail/message.go @@ -79,7 +79,7 @@ func buildDateLayouts() { years := [...]string{"2006", "06"} // year = 4*DIGIT / 2*DIGIT seconds := [...]string{":05", ""} // second // "-0700 (MST)" is not in RFC 5322, but is common. - zones := [...]string{"-0700", "MST", "-0700 (MST)"} // zone = (("+" / "-") 4DIGIT) / "GMT" / ... + zones := [...]string{"-0700", "MST"} // zone = (("+" / "-") 4DIGIT) / "GMT" / ... for _, dow := range dows { for _, day := range days { @@ -98,6 +98,29 @@ func buildDateLayouts() { // ParseDate parses an RFC 5322 date string. func ParseDate(date string) (time.Time, error) { dateLayoutsBuildOnce.Do(buildDateLayouts) + // CR and LF must match and are tolerated anywhere in the date field. + date = strings.ReplaceAll(date, "\r\n", "") + if strings.Index(date, "\r") != -1 { + return time.Time{}, errors.New("mail: header has a CR without LF") + } + // Re-using some addrParser methods which support obsolete text, i.e. non-printable ASCII + p := addrParser{date, nil} + p.skipSpace() + + // RFC 5322: zone = (FWS ( "+" / "-" ) 4DIGIT) / obs-zone + // zone length is always 5 chars unless obsolete (obs-zone) + if ind := strings.IndexAny(p.s, "+-"); ind != -1 && len(p.s) >= ind+5 { + date = p.s[:ind+5] + p.s = p.s[ind+5:] + } else if ind := strings.Index(p.s, "T"); ind != -1 && len(p.s) >= ind+1 { + // The last letter T of the obsolete time zone is checked when no standard time zone is found. + // If T is misplaced, the date to parse is garbage. + date = p.s[:ind+1] + p.s = p.s[ind+1:] + } + if !p.skipCFWS() { + return time.Time{}, errors.New("mail: misformatted parenthetical comment") + } for _, layout := range dateLayouts { t, err := time.Parse(layout, date) if err == nil { diff --git a/libgo/go/net/mail/message_test.go b/libgo/go/net/mail/message_test.go index 2950bc4..acab538 100644 --- a/libgo/go/net/mail/message_test.go +++ b/libgo/go/net/mail/message_test.go @@ -124,6 +124,157 @@ func TestDateParsing(t *testing.T) { } } +func TestDateParsingCFWS(t *testing.T) { + tests := []struct { + dateStr string + exp time.Time + valid bool + }{ + // FWS-only. No date. + { + " ", + // nil is not allowed + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + false, + }, + // FWS is allowed before optional day of week. + { + " Fri, 21 Nov 1997 09:55:06 -0600", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + true, + }, + { + "21 Nov 1997 09:55:06 -0600", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + true, + }, + { + "Fri 21 Nov 1997 09:55:06 -0600", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + false, // missing , + }, + // FWS is allowed before day of month but HTAB fails. + { + "Fri, 21 Nov 1997 09:55:06 -0600", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + true, + }, + // FWS is allowed before and after year but HTAB fails. + { + "Fri, 21 Nov 1997 09:55:06 -0600", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + true, + }, + // FWS is allowed before zone but HTAB is not handled. Obsolete timezone is handled. + { + "Fri, 21 Nov 1997 09:55:06 CST", + time.Time{}, + true, + }, + // FWS is allowed after date and a CRLF is already replaced. + { + "Fri, 21 Nov 1997 09:55:06 CST (no leading FWS and a trailing CRLF) \r\n", + time.Time{}, + true, + }, + // CFWS is a reduced set of US-ASCII where space and accentuated are obsolete. No error. + { + "Fri, 21 Nov 1997 09:55:06 -0600 (MDT and non-US-ASCII signs éèç )", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + true, + }, + // CFWS is allowed after zone including a nested comment. + // Trailing FWS is allowed. + { + "Fri, 21 Nov 1997 09:55:06 -0600 \r\n (thisisa(valid)cfws) \t ", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + true, + }, + // CRLF is incomplete and misplaced. + { + "Fri, 21 Nov 1997 \r 09:55:06 -0600 \r\n (thisisa(valid)cfws) \t ", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + false, + }, + // CRLF is complete but misplaced. No error is returned. + { + "Fri, 21 Nov 199\r\n7 09:55:06 -0600 \r\n (thisisa(valid)cfws) \t ", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + true, // should be false in the strict interpretation of RFC 5322. + }, + // Invalid ASCII in date. + { + "Fri, 21 Nov 1997 ù 09:55:06 -0600 \r\n (thisisa(valid)cfws) \t ", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + false, + }, + // CFWS chars () in date. + { + "Fri, 21 Nov () 1997 09:55:06 -0600 \r\n (thisisa(valid)cfws) \t ", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + false, + }, + // Timezone is invalid but T is found in comment. + { + "Fri, 21 Nov 1997 09:55:06 -060 \r\n (Thisisa(valid)cfws) \t ", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + false, + }, + // Date has no month. + { + "Fri, 21 1997 09:55:06 -0600", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + false, + }, + // Invalid month : OCT iso Oct + { + "Fri, 21 OCT 1997 09:55:06 CST", + time.Time{}, + false, + }, + // A too short time zone. + { + "Fri, 21 Nov 1997 09:55:06 -060", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + false, + }, + // A too short obsolete time zone. + { + "Fri, 21 1997 09:55:06 GT", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + false, + }, + } + for _, test := range tests { + hdr := Header{ + "Date": []string{test.dateStr}, + } + date, err := hdr.Date() + if err != nil && test.valid { + t.Errorf("Header(Date: %s).Date(): %v", test.dateStr, err) + } else if err == nil && test.exp.IsZero() { + // OK. Used when exact result depends on the + // system's local zoneinfo. + } else if err == nil && !date.Equal(test.exp) && test.valid { + t.Errorf("Header(Date: %s).Date() = %+v, want %+v", test.dateStr, date, test.exp) + } else if err == nil && !test.valid { // an invalid expression was tested + t.Errorf("Header(Date: %s).Date() did not return an error but %v", test.dateStr, date) + } + + date, err = ParseDate(test.dateStr) + if err != nil && test.valid { + t.Errorf("ParseDate(%s): %v", test.dateStr, err) + } else if err == nil && test.exp.IsZero() { + // OK. Used when exact result depends on the + // system's local zoneinfo. + } else if err == nil && !test.valid { // an invalid expression was tested + t.Errorf("ParseDate(%s) did not return an error but %v", test.dateStr, date) + } else if err == nil && test.valid && !date.Equal(test.exp) { + t.Errorf("ParseDate(%s) = %+v, want %+v", test.dateStr, date, test.exp) + } + } +} + func TestAddressParsingError(t *testing.T) { mustErrTestCases := [...]struct { text string diff --git a/libgo/go/net/main_conf_test.go b/libgo/go/net/main_conf_test.go index b535046..a92dff5 100644 --- a/libgo/go/net/main_conf_test.go +++ b/libgo/go/net/main_conf_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !js,!nacl,!plan9,!windows +// +build !js,!plan9,!windows package net diff --git a/libgo/go/net/main_noconf_test.go b/libgo/go/net/main_noconf_test.go index 55e3770..bac84aa 100644 --- a/libgo/go/net/main_noconf_test.go +++ b/libgo/go/net/main_noconf_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build js,wasm nacl plan9 windows +// +build js,wasm plan9 windows package net diff --git a/libgo/go/net/main_unix_test.go b/libgo/go/net/main_unix_test.go index af9f2d4..ef7e915 100644 --- a/libgo/go/net/main_unix_test.go +++ b/libgo/go/net/main_unix_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd linux nacl netbsd openbsd solaris +// +build aix darwin dragonfly freebsd hurd linux netbsd openbsd solaris package net diff --git a/libgo/go/net/net_test.go b/libgo/go/net/net_test.go index 463ae88..a740674 100644 --- a/libgo/go/net/net_test.go +++ b/libgo/go/net/net_test.go @@ -72,7 +72,7 @@ func TestCloseRead(t *testing.T) { func TestCloseWrite(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } @@ -285,7 +285,6 @@ func TestPacketConnClose(t *testing.T) { } } -// nacl was previous failing to reuse an address. func TestListenCloseListen(t *testing.T) { const maxTries = 10 for tries := 0; tries < maxTries; tries++ { @@ -302,7 +301,7 @@ func TestListenCloseListen(t *testing.T) { } ln, err = Listen("tcp", addr) if err == nil { - // Success. nacl couldn't do this before. + // Success. (This test didn't always make it here earlier.) ln.Close() return } @@ -541,7 +540,7 @@ func TestNotTemporaryRead(t *testing.T) { if err == nil { return errors.New("Read succeeded unexpectedly") } else if err == io.EOF { - // This happens on NaCl and Plan 9. + // This happens on Plan 9. return nil } else if ne, ok := err.(Error); !ok { return fmt.Errorf("unexpected error %v", err) diff --git a/libgo/go/net/platform_test.go b/libgo/go/net/platform_test.go index 10f55c9..d35dfaa 100644 --- a/libgo/go/net/platform_test.go +++ b/libgo/go/net/platform_test.go @@ -37,13 +37,9 @@ func testableNetwork(network string) bool { ss := strings.Split(network, ":") switch ss[0] { case "ip+nopriv": - switch runtime.GOOS { - case "nacl": - return false - } case "ip", "ip4", "ip6": switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": return false default: if os.Getuid() != 0 { @@ -52,7 +48,7 @@ func testableNetwork(network string) bool { } case "unix", "unixgram": switch runtime.GOOS { - case "android", "nacl", "plan9", "windows": + case "android", "plan9", "windows": return false case "aix": return unixEnabledOnAIX @@ -63,7 +59,7 @@ func testableNetwork(network string) bool { } case "unixpacket": switch runtime.GOOS { - case "aix", "android", "darwin", "nacl", "plan9", "windows": + case "aix", "android", "darwin", "plan9", "windows": return false case "netbsd": // It passes on amd64 at least. 386 fails (Issue 22927). arm is unknown. diff --git a/libgo/go/net/port_unix.go b/libgo/go/net/port_unix.go index 6da6f5a..a3b402f 100644 --- a/libgo/go/net/port_unix.go +++ b/libgo/go/net/port_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris nacl +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris // Read system port mappings from /etc/services diff --git a/libgo/go/net/rawconn.go b/libgo/go/net/rawconn.go index c40ea4a..c786354 100644 --- a/libgo/go/net/rawconn.go +++ b/libgo/go/net/rawconn.go @@ -15,7 +15,7 @@ import ( // deadlines. If the user-provided callback returns false, the Write // method will fail immediately. -// BUG(mikio): On JS, NaCl and Plan 9, the Control, Read and Write +// BUG(mikio): On JS and Plan 9, the Control, Read and Write // methods of syscall.RawConn are not implemented. type rawConn struct { diff --git a/libgo/go/net/rawconn_stub_test.go b/libgo/go/net/rawconn_stub_test.go index 0a033c1..cec977f 100644 --- a/libgo/go/net/rawconn_stub_test.go +++ b/libgo/go/net/rawconn_stub_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build js,wasm nacl plan9 +// +build js,wasm plan9 package net diff --git a/libgo/go/net/rawconn_test.go b/libgo/go/net/rawconn_test.go index 11900df..9a82f8f 100644 --- a/libgo/go/net/rawconn_test.go +++ b/libgo/go/net/rawconn_test.go @@ -15,7 +15,7 @@ import ( func TestRawConnReadWrite(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } @@ -175,7 +175,7 @@ func TestRawConnReadWrite(t *testing.T) { func TestRawConnControl(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } diff --git a/libgo/go/net/sendfile_stub.go b/libgo/go/net/sendfile_stub.go index 6d338da..53bc53a 100644 --- a/libgo/go/net/sendfile_stub.go +++ b/libgo/go/net/sendfile_stub.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin js,wasm nacl netbsd openbsd +// +build aix darwin js,wasm netbsd openbsd package net diff --git a/libgo/go/net/sendfile_test.go b/libgo/go/net/sendfile_test.go index 911e613..13842a1 100644 --- a/libgo/go/net/sendfile_test.go +++ b/libgo/go/net/sendfile_test.go @@ -218,7 +218,7 @@ func TestSendfileSeeked(t *testing.T) { // Test that sendfile doesn't put a pipe into blocking mode. func TestSendfilePipe(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9", "windows": + case "plan9", "windows": // These systems don't support deadlines on pipes. t.Skipf("skipping on %s", runtime.GOOS) } diff --git a/libgo/go/net/sendfile_windows.go b/libgo/go/net/sendfile_windows.go index bccd8b1..59b1b0d 100644 --- a/libgo/go/net/sendfile_windows.go +++ b/libgo/go/net/sendfile_windows.go @@ -18,10 +18,8 @@ import ( // non-EOF error. // // if handled == false, sendFile performed no work. -// -// Note that sendfile for windows does not support >2GB file. func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) { - var n int64 = 0 // by default, copy until EOF + var n int64 = 0 // by default, copy until EOF. lr, ok := r.(*io.LimitedReader) if ok { @@ -30,18 +28,20 @@ func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) { return 0, nil, true } } + f, ok := r.(*os.File) if !ok { return 0, nil, false } - done, err := poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), n) - + written, err = poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), n) if err != nil { - return 0, wrapSyscallError("transmitfile", err), false + err = wrapSyscallError("transmitfile", err) } - if lr != nil { - lr.N -= int64(done) - } - return int64(done), nil, true + + // If any byte was copied, regardless of any error + // encountered mid-way, handled must be set to true. + handled = written > 0 + + return } diff --git a/libgo/go/net/server_test.go b/libgo/go/net/server_test.go index 1608beb..2673b87 100644 --- a/libgo/go/net/server_test.go +++ b/libgo/go/net/server_test.go @@ -56,71 +56,79 @@ func TestTCPServer(t *testing.T) { const N = 3 for i, tt := range tcpServerTests { - if !testableListenArgs(tt.snet, tt.saddr, tt.taddr) { - t.Logf("skipping %s test", tt.snet+" "+tt.saddr+"<-"+tt.taddr) - continue - } - - ln, err := Listen(tt.snet, tt.saddr) - if err != nil { - if perr := parseDialError(err); perr != nil { - t.Error(perr) + t.Run(tt.snet+" "+tt.saddr+"<-"+tt.taddr, func(t *testing.T) { + if !testableListenArgs(tt.snet, tt.saddr, tt.taddr) { + t.Skip("not testable") } - t.Fatal(err) - } - var lss []*localServer - var tpchs []chan error - defer func() { - for _, ls := range lss { - ls.teardown() - } - }() - for i := 0; i < N; i++ { - ls, err := (&streamListener{Listener: ln}).newLocalServer() + ln, err := Listen(tt.snet, tt.saddr) if err != nil { + if perr := parseDialError(err); perr != nil { + t.Error(perr) + } t.Fatal(err) } - lss = append(lss, ls) - tpchs = append(tpchs, make(chan error, 1)) - } - for i := 0; i < N; i++ { - ch := tpchs[i] - handler := func(ls *localServer, ln Listener) { transponder(ln, ch) } - if err := lss[i].buildup(handler); err != nil { - t.Fatal(err) - } - } - var trchs []chan error - for i := 0; i < N; i++ { - _, port, err := SplitHostPort(lss[i].Listener.Addr().String()) - if err != nil { - t.Fatal(err) + var lss []*localServer + var tpchs []chan error + defer func() { + for _, ls := range lss { + ls.teardown() + } + }() + for i := 0; i < N; i++ { + ls, err := (&streamListener{Listener: ln}).newLocalServer() + if err != nil { + t.Fatal(err) + } + lss = append(lss, ls) + tpchs = append(tpchs, make(chan error, 1)) } - d := Dialer{Timeout: someTimeout} - c, err := d.Dial(tt.tnet, JoinHostPort(tt.taddr, port)) - if err != nil { - if perr := parseDialError(err); perr != nil { - t.Error(perr) + for i := 0; i < N; i++ { + ch := tpchs[i] + handler := func(ls *localServer, ln Listener) { transponder(ln, ch) } + if err := lss[i].buildup(handler); err != nil { + t.Fatal(err) } - t.Fatal(err) } - defer c.Close() - trchs = append(trchs, make(chan error, 1)) - go transceiver(c, []byte("TCP SERVER TEST"), trchs[i]) - } - for _, ch := range trchs { - for err := range ch { - t.Errorf("#%d: %v", i, err) + var trchs []chan error + for i := 0; i < N; i++ { + _, port, err := SplitHostPort(lss[i].Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + d := Dialer{Timeout: someTimeout} + c, err := d.Dial(tt.tnet, JoinHostPort(tt.taddr, port)) + if err != nil { + if perr := parseDialError(err); perr != nil { + t.Error(perr) + } + if tt.taddr == "::1" && os.Getenv("GO_BUILDER_NAME") == "darwin-amd64-10_12" && os.IsTimeout(err) { + // A suspected kernel bug in macOS 10.12 occasionally results in + // "i/o timeout" errors when dialing address ::1. The errors have not + // been observed on newer versions of the OS, so we don't plan to work + // around them. See https://golang.org/issue/32919. + t.Skipf("skipping due to error on known-flaky macOS 10.12 builder: %v", err) + } + t.Fatal(err) + } + defer c.Close() + trchs = append(trchs, make(chan error, 1)) + go transceiver(c, []byte("TCP SERVER TEST"), trchs[i]) } - } - for _, ch := range tpchs { - for err := range ch { - t.Errorf("#%d: %v", i, err) + + for _, ch := range trchs { + for err := range ch { + t.Errorf("#%d: %v", i, err) + } } - } + for _, ch := range tpchs { + for err := range ch { + t.Errorf("#%d: %v", i, err) + } + } + }) } } diff --git a/libgo/go/net/smtp/smtp_test.go b/libgo/go/net/smtp/smtp_test.go index 8195f91..cfda079 100644 --- a/libgo/go/net/smtp/smtp_test.go +++ b/libgo/go/net/smtp/smtp_test.go @@ -9,13 +9,13 @@ import ( "bytes" "crypto/tls" "crypto/x509" + "fmt" "internal/testenv" "io" "net" "net/textproto" "runtime" "strings" - "sync" "testing" "time" ) @@ -642,13 +642,13 @@ func TestSendMailWithAuth(t *testing.T) { t.Fatalf("Unable to create listener: %v", err) } defer l.Close() - wg := sync.WaitGroup{} - var done = make(chan struct{}) + + errCh := make(chan error) go func() { - defer wg.Done() + defer close(errCh) conn, err := l.Accept() if err != nil { - t.Errorf("Accept error: %v", err) + errCh <- fmt.Errorf("Accept: %v", err) return } defer conn.Close() @@ -656,13 +656,21 @@ func TestSendMailWithAuth(t *testing.T) { tc := textproto.NewConn(conn) tc.PrintfLine("220 hello world") msg, err := tc.ReadLine() - if msg == "EHLO localhost" { - tc.PrintfLine("250 mx.google.com at your service") + if err != nil { + errCh <- fmt.Errorf("ReadLine error: %v", err) + return + } + const wantMsg = "EHLO localhost" + if msg != wantMsg { + errCh <- fmt.Errorf("unexpected response %q; want %q", msg, wantMsg) + return + } + err = tc.PrintfLine("250 mx.google.com at your service") + if err != nil { + errCh <- fmt.Errorf("PrintfLine: %v", err) + return } - // for this test case, there should have no more traffic - <-done }() - wg.Add(1) err = SendMail(l.Addr().String(), PlainAuth("", "user", "pass", "smtp.google.com"), "test@example.com", []string{"other@example.com"}, []byte(strings.Replace(`From: test@example.com To: other@example.com @@ -676,8 +684,10 @@ SendMail is working for me. if err.Error() != "smtp: server doesn't support AUTH" { t.Errorf("Expected: smtp: server doesn't support AUTH, got: %s", err) } - close(done) - wg.Wait() + err = <-errCh + if err != nil { + t.Fatalf("server error: %v", err) + } } func TestAuthFailed(t *testing.T) { diff --git a/libgo/go/net/sock_posix.go b/libgo/go/net/sock_posix.go index 8076cf1..8fe9bc7 100644 --- a/libgo/go/net/sock_posix.go +++ b/libgo/go/net/sock_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd hurd linux netbsd openbsd solaris windows package net diff --git a/libgo/go/net/sock_stub.go b/libgo/go/net/sock_stub.go index eca1f94..6d44f43 100644 --- a/libgo/go/net/sock_stub.go +++ b/libgo/go/net/sock_stub.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix hurd nacl js,wasm solaris +// +build aix hurd js,wasm solaris package net diff --git a/libgo/go/net/sockaddr_posix.go b/libgo/go/net/sockaddr_posix.go index 9f24650..470d044 100644 --- a/libgo/go/net/sockaddr_posix.go +++ b/libgo/go/net/sockaddr_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris windows package net diff --git a/libgo/go/net/sockopt_stub.go b/libgo/go/net/sockopt_stub.go index bc06675..52624a3 100644 --- a/libgo/go/net/sockopt_stub.go +++ b/libgo/go/net/sockopt_stub.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build nacl js,wasm +// +build js,wasm package net diff --git a/libgo/go/net/sockoptip_stub.go b/libgo/go/net/sockoptip_stub.go index 3297969..57cd289 100644 --- a/libgo/go/net/sockoptip_stub.go +++ b/libgo/go/net/sockoptip_stub.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build nacl js,wasm +// +build js,wasm package net diff --git a/libgo/go/net/sys_cloexec.go b/libgo/go/net/sys_cloexec.go index e97fb21..89aad70 100644 --- a/libgo/go/net/sys_cloexec.go +++ b/libgo/go/net/sys_cloexec.go @@ -5,7 +5,7 @@ // This file implements sysSocket and accept for platforms that do not // provide a fast path for setting SetNonblock and CloseOnExec. -// +build aix darwin nacl solaris +// +build aix darwin solaris package net diff --git a/libgo/go/net/tcpsock.go b/libgo/go/net/tcpsock.go index 0daa2f6..9a9b03a 100644 --- a/libgo/go/net/tcpsock.go +++ b/libgo/go/net/tcpsock.go @@ -12,7 +12,7 @@ import ( "time" ) -// BUG(mikio): On JS, NaCl and Windows, the File method of TCPConn and +// BUG(mikio): On JS and Windows, the File method of TCPConn and // TCPListener is not implemented. // TCPAddr represents the address of a TCP end point. @@ -337,3 +337,8 @@ func ListenTCP(network string, laddr *TCPAddr) (*TCPListener, error) { } return ln, nil } + +// roundDurationUp rounds d to the next multiple of to. +func roundDurationUp(d time.Duration, to time.Duration) time.Duration { + return (d + to - 1) / to +} diff --git a/libgo/go/net/tcpsock_plan9.go b/libgo/go/net/tcpsock_plan9.go index e2e8359..768d03b 100644 --- a/libgo/go/net/tcpsock_plan9.go +++ b/libgo/go/net/tcpsock_plan9.go @@ -23,7 +23,12 @@ func (sd *sysDialer) dialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPCo func (sd *sysDialer) doDialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPConn, error) { switch sd.network { - case "tcp", "tcp4", "tcp6": + case "tcp4": + // Plan 9 doesn't complain about [::]:0->127.0.0.1, so it's up to us. + if laddr != nil && len(laddr.IP) != 0 && laddr.IP.To4() == nil { + return nil, &AddrError{Err: "non-IPv4 local address", Addr: laddr.String()} + } + case "tcp", "tcp6": default: return nil, UnknownNetworkError(sd.network) } diff --git a/libgo/go/net/tcpsock_posix.go b/libgo/go/net/tcpsock_posix.go index 1ff0ec0..2c359da 100644 --- a/libgo/go/net/tcpsock_posix.go +++ b/libgo/go/net/tcpsock_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris windows package net diff --git a/libgo/go/net/tcpsock_test.go b/libgo/go/net/tcpsock_test.go index a89f621..920bf42 100644 --- a/libgo/go/net/tcpsock_test.go +++ b/libgo/go/net/tcpsock_test.go @@ -480,10 +480,6 @@ func TestTCPReadWriteAllocs(t *testing.T) { // I/O on Plan 9 allocates memory. // See net/fd_io_plan9.go. t.Skipf("not supported on %s", runtime.GOOS) - case "nacl": - // NaCl needs to allocate pseudo file descriptor - // stuff. See syscall/fd_nacl.go. - t.Skipf("not supported on %s", runtime.GOOS) } ln, err := Listen("tcp", "127.0.0.1:0") diff --git a/libgo/go/net/tcpsockopt_darwin.go b/libgo/go/net/tcpsockopt_darwin.go index da0d173..53c6756 100644 --- a/libgo/go/net/tcpsockopt_darwin.go +++ b/libgo/go/net/tcpsockopt_darwin.go @@ -15,8 +15,7 @@ const sysTCP_KEEPINTVL = 0x101 func setKeepAlivePeriod(fd *netFD, d time.Duration) error { // The kernel expects seconds so round to next highest second. - d += (time.Second - time.Nanosecond) - secs := int(d.Seconds()) + secs := int(roundDurationUp(d, time.Second)) if err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, sysTCP_KEEPINTVL, secs); err != nil { return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/tcpsockopt_dragonfly.go b/libgo/go/net/tcpsockopt_dragonfly.go index 2b018f2..b473c02 100644 --- a/libgo/go/net/tcpsockopt_dragonfly.go +++ b/libgo/go/net/tcpsockopt_dragonfly.go @@ -13,8 +13,7 @@ import ( func setKeepAlivePeriod(fd *netFD, d time.Duration) error { // The kernel expects milliseconds so round to next highest // millisecond. - d += (time.Millisecond - time.Nanosecond) - msecs := int(d / time.Millisecond) + msecs := int(roundDurationUp(d, time.Millisecond)) if err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, msecs); err != nil { return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/tcpsockopt_solaris.go b/libgo/go/net/tcpsockopt_solaris.go index aa86a29..f15e589 100644 --- a/libgo/go/net/tcpsockopt_solaris.go +++ b/libgo/go/net/tcpsockopt_solaris.go @@ -11,11 +11,22 @@ import ( ) func setKeepAlivePeriod(fd *netFD, d time.Duration) error { - // The kernel expects seconds so round to next highest second. - d += (time.Second - time.Nanosecond) - secs := int(d.Seconds()) + // The kernel expects milliseconds so round to next highest + // millisecond. + msecs := int(roundDurationUp(d, time.Millisecond)) - err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.SO_KEEPALIVE, secs) + // Normally we'd do + // syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, secs) + // here, but we can't because Solaris does not have TCP_KEEPINTVL. + // Solaris has TCP_KEEPALIVE_ABORT_THRESHOLD, but it's not the same + // thing, it refers to the total time until aborting (not between + // probes), and it uses an exponential backoff algorithm instead of + // waiting the same time between probes. We can't hope for the best + // and do it anyway, like on Darwin, because Solaris might eventually + // allocate a constant with a different meaning for the value of + // TCP_KEEPINTVL on illumos. + + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.TCP_KEEPALIVE_THRESHOLD, msecs) runtime.KeepAlive(fd) return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/tcpsockopt_stub.go b/libgo/go/net/tcpsockopt_stub.go index fd7f579..d043da1 100644 --- a/libgo/go/net/tcpsockopt_stub.go +++ b/libgo/go/net/tcpsockopt_stub.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build nacl js,wasm +// +build js,wasm package net diff --git a/libgo/go/net/tcpsockopt_unix.go b/libgo/go/net/tcpsockopt_unix.go index 13cab6c..e05cb73 100644 --- a/libgo/go/net/tcpsockopt_unix.go +++ b/libgo/go/net/tcpsockopt_unix.go @@ -14,8 +14,7 @@ import ( func setKeepAlivePeriod(fd *netFD, d time.Duration) error { // The kernel expects seconds so round to next highest second. - d += (time.Second - time.Nanosecond) - secs := int(d.Seconds()) + secs := int(roundDurationUp(d, time.Second)) if err := fd.pfd.SetsockoptInt(syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, secs); err != nil { return wrapSyscallError("setsockopt", err) } diff --git a/libgo/go/net/tcpsockopt_windows.go b/libgo/go/net/tcpsockopt_windows.go index 73dead1..4a0b094 100644 --- a/libgo/go/net/tcpsockopt_windows.go +++ b/libgo/go/net/tcpsockopt_windows.go @@ -15,8 +15,7 @@ import ( func setKeepAlivePeriod(fd *netFD, d time.Duration) error { // The kernel expects milliseconds so round to next highest // millisecond. - d += (time.Millisecond - time.Nanosecond) - msecs := uint32(d / time.Millisecond) + msecs := uint32(roundDurationUp(d, time.Millisecond)) ka := syscall.TCPKeepalive{ OnOff: 1, Time: msecs, diff --git a/libgo/go/net/textproto/header.go b/libgo/go/net/textproto/header.go index ed096d9..a58df7a 100644 --- a/libgo/go/net/textproto/header.go +++ b/libgo/go/net/textproto/header.go @@ -26,8 +26,7 @@ func (h MIMEHeader) Set(key, value string) { // It is case insensitive; CanonicalMIMEHeaderKey is used // to canonicalize the provided key. // If there are no values associated with the key, Get returns "". -// To access multiple values of a key, or to use non-canonical keys, -// access the map directly. +// To use non-canonical keys, access the map directly. func (h MIMEHeader) Get(key string) string { if h == nil { return "" @@ -39,6 +38,18 @@ func (h MIMEHeader) Get(key string) string { return v[0] } +// Values returns all values associated with the given key. +// It is case insensitive; CanonicalMIMEHeaderKey is +// used to canonicalize the provided key. To use non-canonical +// keys, access the map directly. +// The returned slice is not a copy. +func (h MIMEHeader) Values(key string) []string { + if h == nil { + return nil + } + return h[CanonicalMIMEHeaderKey(key)] +} + // Del deletes the values associated with key. func (h MIMEHeader) Del(key string) { delete(h, CanonicalMIMEHeaderKey(key)) diff --git a/libgo/go/net/textproto/header_test.go b/libgo/go/net/textproto/header_test.go new file mode 100644 index 0000000..de9405c --- /dev/null +++ b/libgo/go/net/textproto/header_test.go @@ -0,0 +1,54 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +import "testing" + +type canonicalHeaderKeyTest struct { + in, out string +} + +var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{ + {"a-b-c", "A-B-C"}, + {"a-1-c", "A-1-C"}, + {"User-Agent", "User-Agent"}, + {"uSER-aGENT", "User-Agent"}, + {"user-agent", "User-Agent"}, + {"USER-AGENT", "User-Agent"}, + + // Other valid tchar bytes in tokens: + {"foo-bar_baz", "Foo-Bar_baz"}, + {"foo-bar$baz", "Foo-Bar$baz"}, + {"foo-bar~baz", "Foo-Bar~baz"}, + {"foo-bar*baz", "Foo-Bar*baz"}, + + // Non-ASCII or anything with spaces or non-token chars is unchanged: + {"üser-agenT", "üser-agenT"}, + {"a B", "a B"}, + + // This caused a panic due to mishandling of a space: + {"C Ontent-Transfer-Encoding", "C Ontent-Transfer-Encoding"}, + {"foo bar", "foo bar"}, +} + +func TestCanonicalMIMEHeaderKey(t *testing.T) { + for _, tt := range canonicalHeaderKeyTests { + if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out { + t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out) + } + } +} + +// Issue #34799 add a Header method to get multiple values []string, with canonicalized key +func TestMIMEHeaderMultipleValues(t *testing.T) { + testHeader := MIMEHeader{ + "Set-Cookie": {"cookie 1", "cookie 2"}, + } + values := testHeader.Values("set-cookie") + n := len(values) + if n != 2 { + t.Errorf("count: %d; want 2", n) + } +} diff --git a/libgo/go/net/textproto/reader.go b/libgo/go/net/textproto/reader.go index a5cab99..a505da9 100644 --- a/libgo/go/net/textproto/reader.go +++ b/libgo/go/net/textproto/reader.go @@ -7,6 +7,7 @@ package textproto import ( "bufio" "bytes" + "fmt" "io" "io/ioutil" "strconv" @@ -90,7 +91,7 @@ func (r *Reader) readLineSlice() ([]byte, error) { // A line consisting of only white space is never continued. // func (r *Reader) ReadContinuedLine() (string, error) { - line, err := r.readContinuedLineSlice() + line, err := r.readContinuedLineSlice(noValidation) return string(line), err } @@ -111,7 +112,7 @@ func trim(s []byte) []byte { // ReadContinuedLineBytes is like ReadContinuedLine but // returns a []byte instead of a string. func (r *Reader) ReadContinuedLineBytes() ([]byte, error) { - line, err := r.readContinuedLineSlice() + line, err := r.readContinuedLineSlice(noValidation) if line != nil { buf := make([]byte, len(line)) copy(buf, line) @@ -120,7 +121,15 @@ func (r *Reader) ReadContinuedLineBytes() ([]byte, error) { return line, err } -func (r *Reader) readContinuedLineSlice() ([]byte, error) { +// readContinuedLineSlice reads continued lines from the reader buffer, +// returning a byte slice with all lines. The validateFirstLine function +// is run on the first read line, and if it returns an error then this +// error is returned from readContinuedLineSlice. +func (r *Reader) readContinuedLineSlice(validateFirstLine func([]byte) error) ([]byte, error) { + if validateFirstLine == nil { + return nil, fmt.Errorf("missing validateFirstLine func") + } + // Read the first line. line, err := r.readLineSlice() if err != nil { @@ -130,6 +139,10 @@ func (r *Reader) readContinuedLineSlice() ([]byte, error) { return line, nil } + if err := validateFirstLine(line); err != nil { + return nil, err + } + // Optimistically assume that we have started to buffer the next line // and it starts with an ASCII letter (the next header key), or a blank // line, so we can avoid copying that buffered data around in memory @@ -490,23 +503,17 @@ func (r *Reader) ReadMIMEHeader() (MIMEHeader, error) { } for { - kv, err := r.readContinuedLineSlice() + kv, err := r.readContinuedLineSlice(mustHaveFieldNameColon) if len(kv) == 0 { return m, err } - // Key ends at first colon; should not have trailing spaces - // but they appear in the wild, violating specs, so we remove - // them if present. + // Key ends at first colon. i := bytes.IndexByte(kv, ':') if i < 0 { return m, ProtocolError("malformed MIME header line: " + string(kv)) } - endKey := i - for endKey > 0 && kv[endKey-1] == ' ' { - endKey-- - } - key := canonicalMIMEHeaderKey(kv[:endKey]) + key := canonicalMIMEHeaderKey(kv[:i]) // As per RFC 7230 field-name is a token, tokens consist of one or more chars. // We could return a ProtocolError here, but better to be liberal in what we @@ -541,6 +548,20 @@ func (r *Reader) ReadMIMEHeader() (MIMEHeader, error) { } } +// noValidation is a no-op validation func for readContinuedLineSlice +// that permits any lines. +func noValidation(_ []byte) error { return nil } + +// mustHaveFieldNameColon ensures that, per RFC 7230, the +// field-name is on a single line, so the first line must +// contain a colon. +func mustHaveFieldNameColon(line []byte) error { + if bytes.IndexByte(line, ':') < 0 { + return ProtocolError(fmt.Sprintf("malformed MIME header: missing colon: %q" + string(line))) + } + return nil +} + // upcomingHeaderNewlines returns an approximation of the number of newlines // that will be in this header. If it gets confused, it returns 0. func (r *Reader) upcomingHeaderNewlines() (n int) { diff --git a/libgo/go/net/textproto/reader_test.go b/libgo/go/net/textproto/reader_test.go index e9ae04d..e0e9adb 100644 --- a/libgo/go/net/textproto/reader_test.go +++ b/libgo/go/net/textproto/reader_test.go @@ -13,41 +13,6 @@ import ( "testing" ) -type canonicalHeaderKeyTest struct { - in, out string -} - -var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{ - {"a-b-c", "A-B-C"}, - {"a-1-c", "A-1-C"}, - {"User-Agent", "User-Agent"}, - {"uSER-aGENT", "User-Agent"}, - {"user-agent", "User-Agent"}, - {"USER-AGENT", "User-Agent"}, - - // Other valid tchar bytes in tokens: - {"foo-bar_baz", "Foo-Bar_baz"}, - {"foo-bar$baz", "Foo-Bar$baz"}, - {"foo-bar~baz", "Foo-Bar~baz"}, - {"foo-bar*baz", "Foo-Bar*baz"}, - - // Non-ASCII or anything with spaces or non-token chars is unchanged: - {"üser-agenT", "üser-agenT"}, - {"a B", "a B"}, - - // This caused a panic due to mishandling of a space: - {"C Ontent-Transfer-Encoding", "C Ontent-Transfer-Encoding"}, - {"foo bar", "foo bar"}, -} - -func TestCanonicalMIMEHeaderKey(t *testing.T) { - for _, tt := range canonicalHeaderKeyTests { - if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out { - t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out) - } - } -} - func reader(s string) *Reader { return NewReader(bufio.NewReader(strings.NewReader(s))) } @@ -188,11 +153,10 @@ func TestLargeReadMIMEHeader(t *testing.T) { } } -// Test that we read slightly-bogus MIME headers seen in the wild, -// with spaces before colons, and spaces in keys. +// TestReadMIMEHeaderNonCompliant checks that we don't normalize headers +// with spaces before colons, and accept spaces in keys. func TestReadMIMEHeaderNonCompliant(t *testing.T) { - // Invalid HTTP response header as sent by an Axis security - // camera: (this is handled by IE, Firefox, Chrome, curl, etc.) + // These invalid headers will be rejected by net/http according to RFC 7230. r := reader("Foo: bar\r\n" + "Content-Language: en\r\n" + "SID : 0\r\n" + @@ -202,9 +166,9 @@ func TestReadMIMEHeaderNonCompliant(t *testing.T) { want := MIMEHeader{ "Foo": {"bar"}, "Content-Language": {"en"}, - "Sid": {"0"}, - "Audio Mode": {"None"}, - "Privilege": {"127"}, + "SID ": {"0"}, + "Audio Mode ": {"None"}, + "Privilege ": {"127"}, } if !reflect.DeepEqual(m, want) || err != nil { t.Fatalf("ReadMIMEHeader =\n%v, %v; want:\n%v", m, err, want) @@ -219,6 +183,10 @@ func TestReadMIMEHeaderMalformed(t *testing.T) { " First: line with leading space\r\nFoo: foo\r\n\r\n", "\tFirst: line with leading tab\r\nFoo: foo\r\n\r\n", "Foo: foo\r\nNo colon second line\r\n\r\n", + "Foo-\n\tBar: foo\r\n\r\n", + "Foo-\r\n\tBar: foo\r\n\r\n", + "Foo\r\n\t: foo\r\n\r\n", + "Foo-\n\tBar", } for _, input := range inputs { diff --git a/libgo/go/net/timeout_test.go b/libgo/go/net/timeout_test.go index b4fc2c0..f54c956 100644 --- a/libgo/go/net/timeout_test.go +++ b/libgo/go/net/timeout_test.go @@ -411,9 +411,6 @@ func TestReadTimeoutMustNotReturn(t *testing.T) { if perr := parseReadError(err); perr != nil { t.Error(perr) } - if err == io.EOF && runtime.GOOS == "nacl" { // see golang.org/issue/8044 - return - } if nerr, ok := err.(Error); !ok || nerr.Timeout() || nerr.Temporary() { t.Fatal(err) } @@ -432,11 +429,6 @@ var readFromTimeoutTests = []struct { } func TestReadFromTimeout(t *testing.T) { - switch runtime.GOOS { - case "nacl": - t.Skipf("not supported on %s", runtime.GOOS) // see golang.org/issue/8916 - } - ch := make(chan Addr) defer close(ch) handler := func(ls *localPacketServer, c PacketConn) { @@ -621,11 +613,6 @@ var writeToTimeoutTests = []struct { func TestWriteToTimeout(t *testing.T) { t.Parallel() - switch runtime.GOOS { - case "nacl": - t.Skipf("not supported on %s", runtime.GOOS) - } - c1, err := newLocalPacketListener("udp") if err != nil { t.Fatal(err) @@ -991,11 +978,6 @@ func TestReadWriteProlongedTimeout(t *testing.T) { func TestReadWriteDeadlineRace(t *testing.T) { t.Parallel() - switch runtime.GOOS { - case "nacl": - t.Skipf("not supported on %s", runtime.GOOS) - } - N := 1000 if testing.Short() { N = 50 @@ -1051,3 +1033,43 @@ func TestReadWriteDeadlineRace(t *testing.T) { }() wg.Wait() // wait for tester goroutine to stop } + +// Issue 35367. +func TestConcurrentSetDeadline(t *testing.T) { + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + + const goroutines = 8 + const conns = 10 + const tries = 100 + + var c [conns]Conn + for i := 0; i < conns; i++ { + c[i], err = Dial(ln.Addr().Network(), ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer c[i].Close() + } + + var wg sync.WaitGroup + wg.Add(goroutines) + now := time.Now() + for i := 0; i < goroutines; i++ { + go func(i int) { + defer wg.Done() + // Make the deadlines steadily earlier, + // to trigger runtime adjusttimers calls. + for j := tries; j > 0; j-- { + for k := 0; k < conns; k++ { + c[k].SetReadDeadline(now.Add(2*time.Hour + time.Duration(i*j*k)*time.Second)) + c[k].SetWriteDeadline(now.Add(1*time.Hour + time.Duration(i*j*k)*time.Second)) + } + } + }(i) + } + wg.Wait() +} diff --git a/libgo/go/net/udpsock.go b/libgo/go/net/udpsock.go index b234ed8..ec2bcfa 100644 --- a/libgo/go/net/udpsock.go +++ b/libgo/go/net/udpsock.go @@ -9,15 +9,12 @@ import ( "syscall" ) -// BUG(mikio): On NaCl and Plan 9, the ReadMsgUDP and +// BUG(mikio): On Plan 9, the ReadMsgUDP and // WriteMsgUDP methods of UDPConn are not implemented. // BUG(mikio): On Windows, the File method of UDPConn is not // implemented. -// BUG(mikio): On NaCl, the ListenMulticastUDP function is not -// implemented. - // BUG(mikio): On JS, methods and functions related to UDPConn are not // implemented. diff --git a/libgo/go/net/udpsock_plan9.go b/libgo/go/net/udpsock_plan9.go index 563d943..79986ce 100644 --- a/libgo/go/net/udpsock_plan9.go +++ b/libgo/go/net/udpsock_plan9.go @@ -109,7 +109,9 @@ func (sl *sysListener) listenUDP(ctx context.Context, laddr *UDPAddr) (*UDPConn, } func (sl *sysListener) listenMulticastUDP(ctx context.Context, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { - l, err := listenPlan9(ctx, sl.network, gaddr) + // Plan 9 does not like announce command with a multicast address, + // so do not specify an IP address when listening. + l, err := listenPlan9(ctx, sl.network, &UDPAddr{IP: nil, Port: gaddr.Port, Zone: gaddr.Zone}) if err != nil { return nil, err } @@ -129,11 +131,13 @@ func (sl *sysListener) listenMulticastUDP(ctx context.Context, ifi *Interface, g return nil, err } } + + have4 := gaddr.IP.To4() != nil for _, addr := range addrs { - if ipnet, ok := addr.(*IPNet); ok { + if ipnet, ok := addr.(*IPNet); ok && (ipnet.IP.To4() != nil) == have4 { _, err = l.ctl.WriteString("addmulti " + ipnet.IP.String() + " " + gaddr.IP.String()) if err != nil { - return nil, err + return nil, &OpError{Op: "addmulti", Net: "", Source: nil, Addr: ipnet, Err: err} } } } diff --git a/libgo/go/net/udpsock_plan9_test.go b/libgo/go/net/udpsock_plan9_test.go index 09f5a5d..3febfcc 100644 --- a/libgo/go/net/udpsock_plan9_test.go +++ b/libgo/go/net/udpsock_plan9_test.go @@ -36,7 +36,7 @@ func TestListenMulticastUDP(t *testing.T) { c1, err := ListenMulticastUDP("udp4", mifc, &UDPAddr{IP: ParseIP("224.0.0.254")}) if err != nil { - t.Fatalf("multicast not working on %s", runtime.GOOS) + t.Fatalf("multicast not working on %s: %v", runtime.GOOS, err) } c1addr := c1.LocalAddr().(*UDPAddr) if err != nil { diff --git a/libgo/go/net/udpsock_posix.go b/libgo/go/net/udpsock_posix.go index 1d4cde1..858a3ef 100644 --- a/libgo/go/net/udpsock_posix.go +++ b/libgo/go/net/udpsock_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris windows package net diff --git a/libgo/go/net/udpsock_test.go b/libgo/go/net/udpsock_test.go index 397b664..947381a 100644 --- a/libgo/go/net/udpsock_test.go +++ b/libgo/go/net/udpsock_test.go @@ -162,13 +162,8 @@ func testWriteToConn(t *testing.T, raddr string) { t.Fatalf("should fail as ErrWriteToConnected: %v", err) } _, _, err = c.(*UDPConn).WriteMsgUDP(b, nil, nil) - switch runtime.GOOS { - case "nacl": // see golang.org/issue/9252 - t.Skipf("not implemented yet on %s", runtime.GOOS) - default: - if err != nil { - t.Fatal(err) - } + if err != nil { + t.Fatal(err) } } @@ -205,13 +200,8 @@ func testWriteToPacketConn(t *testing.T, raddr string) { t.Fatalf("should fail as errMissingAddress: %v", err) } _, _, err = c.(*UDPConn).WriteMsgUDP(b, nil, ra) - switch runtime.GOOS { - case "nacl": // see golang.org/issue/9252 - t.Skipf("not implemented yet on %s", runtime.GOOS) - default: - if err != nil { - t.Fatal(err) - } + if err != nil { + t.Fatal(err) } } @@ -335,7 +325,7 @@ func TestIPv6LinkLocalUnicastUDP(t *testing.T) { func TestUDPZeroBytePayload(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) case "darwin": testenv.SkipFlaky(t, 29225) @@ -373,7 +363,7 @@ func TestUDPZeroBytePayload(t *testing.T) { func TestUDPZeroByteBuffer(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } @@ -410,7 +400,7 @@ func TestUDPZeroByteBuffer(t *testing.T) { func TestUDPReadSizeError(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } diff --git a/libgo/go/net/unixsock.go b/libgo/go/net/unixsock.go index ae912a4..b38438c 100644 --- a/libgo/go/net/unixsock.go +++ b/libgo/go/net/unixsock.go @@ -12,7 +12,7 @@ import ( "time" ) -// BUG(mikio): On JS, NaCl and Plan 9, methods and functions related +// BUG(mikio): On JS and Plan 9, methods and functions related // to UnixConn and UnixListener are not implemented. // BUG(mikio): On Windows, methods and functions related to UnixConn diff --git a/libgo/go/net/unixsock_posix.go b/libgo/go/net/unixsock_posix.go index 68887b4..3721485 100644 --- a/libgo/go/net/unixsock_posix.go +++ b/libgo/go/net/unixsock_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build aix darwin dragonfly freebsd hurd js,wasm linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd hurd js,wasm linux netbsd openbsd solaris windows package net diff --git a/libgo/go/net/unixsock_test.go b/libgo/go/net/unixsock_test.go index 4828990..80cccf2 100644 --- a/libgo/go/net/unixsock_test.go +++ b/libgo/go/net/unixsock_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !js,!nacl,!plan9,!windows +// +build !js,!plan9,!windows package net diff --git a/libgo/go/net/url/url.go b/libgo/go/net/url/url.go index 12ea35f..2880e82 100644 --- a/libgo/go/net/url/url.go +++ b/libgo/go/net/url/url.go @@ -26,7 +26,7 @@ type Error struct { } func (e *Error) Unwrap() error { return e.Err } -func (e *Error) Error() string { return e.Op + " " + e.URL + ": " + e.Err.Error() } +func (e *Error) Error() string { return fmt.Sprintf("%s %q: %s", e.Op, e.URL, e.Err) } func (e *Error) Timeout() bool { t, ok := e.Err.(interface { @@ -42,6 +42,8 @@ func (e *Error) Temporary() bool { return ok && t.Temporary() } +const upperhex = "0123456789ABCDEF" + func ishex(c byte) bool { switch { case '0' <= c && c <= '9': @@ -324,8 +326,8 @@ func escape(s string, mode encoding) string { j++ case shouldEscape(c, mode): t[j] = '%' - t[j+1] = "0123456789ABCDEF"[c>>4] - t[j+2] = "0123456789ABCDEF"[c&15] + t[j+1] = upperhex[c>>4] + t[j+2] = upperhex[c&15] j += 3 default: t[j] = s[i] @@ -449,16 +451,16 @@ func getscheme(rawurl string) (scheme, path string, err error) { return "", rawurl, nil } -// Maybe s is of the form t c u. -// If so, return t, c u (or t, u if cutc == true). -// If not, return s, "". -func split(s string, c string, cutc bool) (string, string) { - i := strings.Index(s, c) +// split slices s into two substrings separated by the first occurrence of +// sep. If cutc is true then sep is excluded from the second substring. +// If sep does not occur in s then s and the empty string is returned. +func split(s string, sep byte, cutc bool) (string, string) { + i := strings.IndexByte(s, sep) if i < 0 { return s, "" } if cutc { - return s[:i], s[i+len(c):] + return s[:i], s[i+1:] } return s[:i], s[i:] } @@ -471,7 +473,7 @@ func split(s string, c string, cutc bool) (string, string) { // error, due to parsing ambiguities. func Parse(rawurl string) (*URL, error) { // Cut off #frag - u, frag := split(rawurl, "#", true) + u, frag := split(rawurl, '#', true) url, err := parse(u, false) if err != nil { return nil, &Error{"parse", u, err} @@ -531,7 +533,7 @@ func parse(rawurl string, viaRequest bool) (*URL, error) { url.ForceQuery = true rest = rest[:len(rest)-1] } else { - rest, url.RawQuery = split(rest, "?", true) + rest, url.RawQuery = split(rest, '?', true) } if !strings.HasPrefix(rest, "/") { @@ -560,7 +562,7 @@ func parse(rawurl string, viaRequest bool) (*URL, error) { if (url.Scheme != "" || !viaRequest && !strings.HasPrefix(rest, "///")) && strings.HasPrefix(rest, "//") { var authority string - authority, rest = split(rest[2:], "/", false) + authority, rest = split(rest[2:], '/', false) url.User, url.Host, err = parseAuthority(authority) if err != nil { return nil, err @@ -599,7 +601,7 @@ func parseAuthority(authority string) (user *Userinfo, host string, err error) { } user = User(userinfo) } else { - username, password := split(userinfo, ":", true) + username, password := split(userinfo, ':', true) if username, err = unescape(username, encodeUserPassword); err != nil { return nil, "", err } @@ -948,8 +950,8 @@ func resolvePath(base, ref string) string { if full == "" { return "" } - var dst []string src := strings.Split(full, "/") + dst := make([]string, 0, len(src)) for _, elem := range src { switch elem { case ".": diff --git a/libgo/go/net/url/url_test.go b/libgo/go/net/url/url_test.go index e83c86c..79fd3d5 100644 --- a/libgo/go/net/url/url_test.go +++ b/libgo/go/net/url/url_test.go @@ -668,6 +668,7 @@ var parseRequestURLTests = []struct { {"foo.html", false}, {"../dir/", false}, + {" http://foo.com", false}, {"http://192.168.0.%31/", false}, {"http://192.168.0.%31:8080/", false}, {"http://[fe80::%31]/", false}, @@ -1429,16 +1430,21 @@ func TestParseErrors(t *testing.T) { {"http://[::1]/", false}, {"http://[::1]a", true}, {"http://[::1]%23", true}, - {"http://[::1%25en0]", false}, // valid zone id - {"http://[::1]:", false}, // colon, but no port OK - {"http://x:", false}, // colon, but no port OK - {"http://[::1]:%38%30", true}, // not allowed: % encoding only for non-ASCII - {"http://[::1%25%41]", false}, // RFC 6874 allows over-escaping in zone - {"http://[%10::1]", true}, // no %xx escapes in IP address - {"http://[::1]/%48", false}, // %xx in path is fine - {"http://%41:8080/", true}, // not allowed: % encoding only for non-ASCII - {"mysql://x@y(z:123)/foo", false}, // golang.org/issue/12023 - {"mysql://x@y(1.2.3.4:123)/foo", false}, + {"http://[::1%25en0]", false}, // valid zone id + {"http://[::1]:", false}, // colon, but no port OK + {"http://x:", false}, // colon, but no port OK + {"http://[::1]:%38%30", true}, // not allowed: % encoding only for non-ASCII + {"http://[::1%25%41]", false}, // RFC 6874 allows over-escaping in zone + {"http://[%10::1]", true}, // no %xx escapes in IP address + {"http://[::1]/%48", false}, // %xx in path is fine + {"http://%41:8080/", true}, // not allowed: % encoding only for non-ASCII + {"mysql://x@y(z:123)/foo", true}, // not well-formed per RFC 3986, golang.org/issue/33646 + {"mysql://x@y(1.2.3.4:123)/foo", true}, + + {" http://foo.com", true}, // invalid character in schema + {"ht tp://foo.com", true}, // invalid character in schema + {"ahttp://foo.com", false}, // valid schema characters + {"1http://foo.com", true}, // invalid character in schema {"http://[]%20%48%54%54%50%2f%31%2e%31%0a%4d%79%48%65%61%64%65%72%3a%20%31%32%33%0a%0a/", true}, // golang.org/issue/11208 {"http://a b.com/", true}, // no space in host name please @@ -1456,7 +1462,7 @@ func TestParseErrors(t *testing.T) { continue } if err != nil { - t.Logf("Parse(%q) = %v; want no error", tt.in, err) + t.Errorf("Parse(%q) = %v; want no error", tt.in, err) } } } @@ -1874,3 +1880,12 @@ func BenchmarkPathUnescape(b *testing.B) { }) } } + +var sink string + +func BenchmarkSplit(b *testing.B) { + url := "http://www.google.com/?q=go+language#foo%26bar" + for i := 0; i < b.N; i++ { + sink, sink = split(url, '#', true) + } +} |