diff options
author | Ian Lance Taylor <ian@gcc.gnu.org> | 2016-07-22 18:15:38 +0000 |
---|---|---|
committer | Ian Lance Taylor <ian@gcc.gnu.org> | 2016-07-22 18:15:38 +0000 |
commit | 22b955cca564a9a3a5b8c9d9dd1e295b7943c128 (patch) | |
tree | abdbd898676e1f853fca2d7e031d105d7ebcf676 /libgo/go/net | |
parent | 9d04a3af4c6491536badf6bde9707c907e4d196b (diff) | |
download | gcc-22b955cca564a9a3a5b8c9d9dd1e295b7943c128.zip gcc-22b955cca564a9a3a5b8c9d9dd1e295b7943c128.tar.gz gcc-22b955cca564a9a3a5b8c9d9dd1e295b7943c128.tar.bz2 |
libgo: update to go1.7rc3
Reviewed-on: https://go-review.googlesource.com/25150
From-SVN: r238662
Diffstat (limited to 'libgo/go/net')
214 files changed, 18769 insertions, 5403 deletions
diff --git a/libgo/go/net/addrselect.go b/libgo/go/net/addrselect.go index 58ab7d7..0b9d160 100644 --- a/libgo/go/net/addrselect.go +++ b/libgo/go/net/addrselect.go @@ -203,7 +203,7 @@ func (s *byRFC6724) Less(i, j int) bool { // (e.g., https://golang.org/issue/13283). Glibc instead only // uses CommonPrefixLen for IPv4 when the source and destination // addresses are on the same subnet, but that requires extra - // work to find the netmask for our source addresses. As a + // work to find the netmask for our source addresses. As a // simpler heuristic, we limit its use to when the source and // destination belong to the same special purpose block. if da4 { diff --git a/libgo/go/net/cgo_android.go b/libgo/go/net/cgo_android.go index fe9925b..ab0368d 100644 --- a/libgo/go/net/cgo_android.go +++ b/libgo/go/net/cgo_android.go @@ -1,4 +1,4 @@ -// Copyright 2014 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_bsd.go b/libgo/go/net/cgo_bsd.go index ae1054b..c1adf20 100644 --- a/libgo/go/net/cgo_bsd.go +++ b/libgo/go/net/cgo_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_linux.go b/libgo/go/net/cgo_linux.go index baf2072..36bac34 100644 --- a/libgo/go/net/cgo_linux.go +++ b/libgo/go/net/cgo_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_netbsd.go b/libgo/go/net/cgo_netbsd.go index 8a16871..1ce0139 100644 --- a/libgo/go/net/cgo_netbsd.go +++ b/libgo/go/net/cgo_netbsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_openbsd.go b/libgo/go/net/cgo_openbsd.go index 1830913..4610246 100644 --- a/libgo/go/net/cgo_openbsd.go +++ b/libgo/go/net/cgo_openbsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_solaris.go b/libgo/go/net/cgo_solaris.go index 05811c6..bd1e8f3 100644 --- a/libgo/go/net/cgo_solaris.go +++ b/libgo/go/net/cgo_solaris.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_stub.go b/libgo/go/net/cgo_stub.go index b86ff7d..5125972 100644 --- a/libgo/go/net/cgo_stub.go +++ b/libgo/go/net/cgo_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -6,6 +6,8 @@ package net +import "context" + func init() { netGo = true } type addrinfoErrno int @@ -14,22 +16,22 @@ func (eai addrinfoErrno) Error() string { return "<nil>" } func (eai addrinfoErrno) Temporary() bool { return false } func (eai addrinfoErrno) Timeout() bool { return false } -func cgoLookupHost(name string) (addrs []string, err error, completed bool) { +func cgoLookupHost(ctx context.Context, name string) (addrs []string, err error, completed bool) { return nil, nil, false } -func cgoLookupPort(network, service string) (port int, err error, completed bool) { +func cgoLookupPort(ctx context.Context, network, service string) (port int, err error, completed bool) { return 0, nil, false } -func cgoLookupIP(name string) (addrs []IPAddr, err error, completed bool) { +func cgoLookupIP(ctx context.Context, name string) (addrs []IPAddr, err error, completed bool) { return nil, nil, false } -func cgoLookupCNAME(name string) (cname string, err error, completed bool) { +func cgoLookupCNAME(ctx context.Context, name string) (cname string, err error, completed bool) { return "", nil, false } -func cgoLookupPTR(addr string) (ptrs []string, err error, completed bool) { +func cgoLookupPTR(ctx context.Context, addr string) (ptrs []string, err error, completed bool) { return nil, nil, false } diff --git a/libgo/go/net/cgo_unix.go b/libgo/go/net/cgo_unix.go index f634323..525c63c 100644 --- a/libgo/go/net/cgo_unix.go +++ b/libgo/go/net/cgo_unix.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -18,6 +18,7 @@ package net */ import ( + "context" "syscall" "unsafe" ) @@ -51,18 +52,31 @@ func (eai addrinfoErrno) Error() string { return bytePtrToString(libc_gai_stre func (eai addrinfoErrno) Temporary() bool { return eai == syscall.EAI_AGAIN } func (eai addrinfoErrno) Timeout() bool { return false } -func cgoLookupHost(name string) (hosts []string, err error, completed bool) { - addrs, err, completed := cgoLookupIP(name) +type portLookupResult struct { + port int + err error +} + +type ipLookupResult struct { + addrs []IPAddr + cname string + err error +} + +type reverseLookupResult struct { + names []string + err error +} + +func cgoLookupHost(ctx context.Context, name string) (hosts []string, err error, completed bool) { + addrs, err, completed := cgoLookupIP(ctx, name) for _, addr := range addrs { hosts = append(hosts, addr.String()) } return } -func cgoLookupPort(network, service string) (port int, err error, completed bool) { - acquireThread() - defer releaseThread() - +func cgoLookupPort(ctx context.Context, network, service string) (port int, err error, completed bool) { var hints syscall.Addrinfo switch network { case "": // no hints @@ -83,11 +97,27 @@ func cgoLookupPort(network, service string) (port int, err error, completed bool hints.Ai_family = syscall.AF_INET6 } } + if ctx.Done() == nil { + port, err := cgoLookupServicePort(&hints, network, service) + return port, err, true + } + result := make(chan portLookupResult, 1) + go cgoPortLookup(result, &hints, network, service) + select { + case r := <-result: + return r.port, r.err, true + case <-ctx.Done(): + // Since there isn't a portable way to cancel the lookup, + // we just let it finish and write to the buffered channel. + return 0, mapErr(ctx.Err()), false + } +} +func cgoLookupServicePort(hints *syscall.Addrinfo, network, service string) (port int, err error) { s := syscall.StringBytePtr(service) var res *syscall.Addrinfo syscall.Entersyscall() - gerrno := libc_getaddrinfo(nil, s, &hints, &res) + gerrno := libc_getaddrinfo(nil, s, hints, &res) syscall.Exitsyscall() if gerrno != 0 { switch gerrno { @@ -100,7 +130,7 @@ func cgoLookupPort(network, service string) (port int, err error, completed bool default: err = addrinfoErrno(gerrno) } - return 0, &DNSError{Err: err.Error(), Name: network + "/" + service}, true + return 0, &DNSError{Err: err.Error(), Name: network + "/" + service} } defer libc_freeaddrinfo(res) @@ -109,17 +139,22 @@ func cgoLookupPort(network, service string) (port int, err error, completed bool case syscall.AF_INET: sa := (*syscall.RawSockaddrInet4)(unsafe.Pointer(r.Ai_addr)) p := (*[2]byte)(unsafe.Pointer(&sa.Port)) - return int(p[0])<<8 | int(p[1]), nil, true + return int(p[0])<<8 | int(p[1]), nil case syscall.AF_INET6: sa := (*syscall.RawSockaddrInet6)(unsafe.Pointer(r.Ai_addr)) p := (*[2]byte)(unsafe.Pointer(&sa.Port)) - return int(p[0])<<8 | int(p[1]), nil, true + return int(p[0])<<8 | int(p[1]), nil } } - return 0, &DNSError{Err: "unknown port", Name: network + "/" + service}, true + return 0, &DNSError{Err: "unknown port", Name: network + "/" + service} } -func cgoLookupIPCNAME(name string) (addrs []IPAddr, cname string, err error, completed bool) { +func cgoPortLookup(result chan<- portLookupResult, hints *syscall.Addrinfo, network, service string) { + port, err := cgoLookupServicePort(hints, network, service) + result <- portLookupResult{port, err} +} + +func cgoLookupIPCNAME(name string) (addrs []IPAddr, cname string, err error) { acquireThread() defer releaseThread() @@ -152,7 +187,7 @@ func cgoLookupIPCNAME(name string) (addrs []IPAddr, cname string, err error, com default: err = addrinfoErrno(gerrno) } - return nil, "", &DNSError{Err: err.Error(), Name: name}, true + return nil, "", &DNSError{Err: err.Error(), Name: name} } defer libc_freeaddrinfo(res) @@ -181,17 +216,42 @@ func cgoLookupIPCNAME(name string) (addrs []IPAddr, cname string, err error, com addrs = append(addrs, addr) } } - return addrs, cname, nil, true + return addrs, cname, nil } -func cgoLookupIP(name string) (addrs []IPAddr, err error, completed bool) { - addrs, _, err, completed = cgoLookupIPCNAME(name) - return +func cgoIPLookup(result chan<- ipLookupResult, name string) { + addrs, cname, err := cgoLookupIPCNAME(name) + result <- ipLookupResult{addrs, cname, err} } -func cgoLookupCNAME(name string) (cname string, err error, completed bool) { - _, cname, err, completed = cgoLookupIPCNAME(name) - return +func cgoLookupIP(ctx context.Context, name string) (addrs []IPAddr, err error, completed bool) { + if ctx.Done() == nil { + addrs, _, err = cgoLookupIPCNAME(name) + return addrs, err, true + } + result := make(chan ipLookupResult, 1) + go cgoIPLookup(result, name) + select { + case r := <-result: + return r.addrs, r.err, true + case <-ctx.Done(): + return nil, mapErr(ctx.Err()), false + } +} + +func cgoLookupCNAME(ctx context.Context, name string) (cname string, err error, completed bool) { + if ctx.Done() == nil { + _, cname, err = cgoLookupIPCNAME(name) + return cname, err, true + } + result := make(chan ipLookupResult, 1) + go cgoIPLookup(result, name) + select { + case r := <-result: + return r.cname, r.err, true + case <-ctx.Done(): + return "", mapErr(ctx.Err()), false + } } // These are roughly enough for the following: @@ -207,10 +267,7 @@ const ( maxNameinfoLen = 4096 ) -func cgoLookupPTR(addr string) ([]string, error, bool) { - acquireThread() - defer releaseThread() - +func cgoLookupPTR(ctx context.Context, addr string) (names []string, err error, completed bool) { var zone string ip := parseIPv4(addr) if ip == nil { @@ -223,9 +280,26 @@ func cgoLookupPTR(addr string) ([]string, error, bool) { if sa == nil { return nil, &DNSError{Err: "invalid address " + ip.String(), Name: addr}, true } - var err error - var b []byte + if ctx.Done() == nil { + names, err := cgoLookupAddrPTR(addr, sa, salen) + return names, err, true + } + result := make(chan reverseLookupResult, 1) + go cgoReverseLookup(result, addr, sa, salen) + select { + case r := <-result: + return r.names, r.err, true + case <-ctx.Done(): + return nil, mapErr(ctx.Err()), false + } +} + +func cgoLookupAddrPTR(addr string, sa *syscall.RawSockaddr, salen syscall.Socklen_t) (names []string, err error) { + acquireThread() + defer releaseThread() + var gerrno int + var b []byte for l := nameinfoLen; l <= maxNameinfoLen; l *= 2 { b = make([]byte, l) gerrno, err = cgoNameinfoPTR(b, sa, salen) @@ -242,16 +316,20 @@ func cgoLookupPTR(addr string) ([]string, error, bool) { default: err = addrinfoErrno(gerrno) } - return nil, &DNSError{Err: err.Error(), Name: addr}, true + return nil, &DNSError{Err: err.Error(), Name: addr} } - for i := 0; i < len(b); i++ { if b[i] == 0 { b = b[:i] break } } - return []string{absDomainName(b)}, nil, true + return []string{absDomainName(b)}, nil +} + +func cgoReverseLookup(result chan<- reverseLookupResult, addr string, sa *syscall.RawSockaddr, salen syscall.Socklen_t) { + names, err := cgoLookupAddrPTR(addr, sa, salen) + result <- reverseLookupResult{names, err} } func cgoSockaddr(ip IP, zone string) (*syscall.RawSockaddr, syscall.Socklen_t) { diff --git a/libgo/go/net/cgo_unix_test.go b/libgo/go/net/cgo_unix_test.go index 4d5ab23..e861c7a 100644 --- a/libgo/go/net/cgo_unix_test.go +++ b/libgo/go/net/cgo_unix_test.go @@ -7,18 +7,76 @@ package net -import "testing" +import ( + "context" + "testing" +) func TestCgoLookupIP(t *testing.T) { - host := "localhost" - _, err, ok := cgoLookupIP(host) + ctx := context.Background() + _, err, ok := cgoLookupIP(ctx, "localhost") if !ok { t.Errorf("cgoLookupIP must not be a placeholder") } if err != nil { t.Error(err) } - if _, err := goLookupIP(host); err != nil { +} + +func TestCgoLookupIPWithCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err, ok := cgoLookupIP(ctx, "localhost") + if !ok { + t.Errorf("cgoLookupIP must not be a placeholder") + } + if err != nil { + t.Error(err) + } +} + +func TestCgoLookupPort(t *testing.T) { + ctx := context.Background() + _, err, ok := cgoLookupPort(ctx, "tcp", "smtp") + if !ok { + t.Errorf("cgoLookupPort must not be a placeholder") + } + if err != nil { + t.Error(err) + } +} + +func TestCgoLookupPortWithCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err, ok := cgoLookupPort(ctx, "tcp", "smtp") + if !ok { + t.Errorf("cgoLookupPort must not be a placeholder") + } + if err != nil { + t.Error(err) + } +} + +func TestCgoLookupPTR(t *testing.T) { + ctx := context.Background() + _, err, ok := cgoLookupPTR(ctx, "127.0.0.1") + if !ok { + t.Errorf("cgoLookupPTR must not be a placeholder") + } + if err != nil { + t.Error(err) + } +} + +func TestCgoLookupPTRWithCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err, ok := cgoLookupPTR(ctx, "127.0.0.1") + if !ok { + t.Errorf("cgoLookupPTR must not be a placeholder") + } + if err != nil { t.Error(err) } } diff --git a/libgo/go/net/conf.go b/libgo/go/net/conf.go index ddaa978..eb72916 100644 --- a/libgo/go/net/conf.go +++ b/libgo/go/net/conf.go @@ -124,16 +124,17 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { print("go package net: hostLookupOrder(", hostname, ") = ", ret.String(), "\n") }() } + fallbackOrder := hostLookupCgo if c.netGo { - return hostLookupFilesDNS + fallbackOrder = hostLookupFilesDNS } if c.forceCgoLookupHost || c.resolv.unknownOpt || c.goos == "android" { - return hostLookupCgo + return fallbackOrder } if byteIndex(hostname, '\\') != -1 || byteIndex(hostname, '%') != -1 { // Don't deal with special form hostnames with backslashes // or '%'. - return hostLookupCgo + return fallbackOrder } // OpenBSD is unique and doesn't use nsswitch.conf. @@ -154,7 +155,7 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { return hostLookupDNSFiles } if len(lookup) < 1 || len(lookup) > 2 { - return hostLookupCgo + return fallbackOrder } switch lookup[0] { case "bind": @@ -162,7 +163,7 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { if lookup[1] == "file" { return hostLookupDNSFiles } - return hostLookupCgo + return fallbackOrder } return hostLookupDNS case "file": @@ -170,11 +171,11 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { if lookup[1] == "bind" { return hostLookupFilesDNS } - return hostLookupCgo + return fallbackOrder } return hostLookupFiles default: - return hostLookupCgo + return fallbackOrder } } @@ -185,11 +186,11 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { hostname = hostname[:len(hostname)-1] } if stringsHasSuffixFold(hostname, ".local") { - // Per RFC 6762, the ".local" TLD is special. And + // Per RFC 6762, the ".local" TLD is special. And // because Go's native resolver doesn't do mDNS or // similar local resolution mechanisms, assume that // libc might (via Avahi, etc) and use cgo. - return hostLookupCgo + return fallbackOrder } nss := c.nss @@ -199,7 +200,7 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { if os.IsNotExist(nss.err) || (nss.err == nil && len(srcs) == 0) { if c.goos == "solaris" { // illumos defaults to "nis [NOTFOUND=return] files" - return hostLookupCgo + return fallbackOrder } if c.goos == "linux" { // glibc says the default is "dns [!UNAVAIL=return] files" @@ -212,21 +213,21 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { // We failed to parse or open nsswitch.conf, so // conservatively assume we should use cgo if it's // available. - return hostLookupCgo + return fallbackOrder } var mdnsSource, filesSource, dnsSource bool var first string for _, src := range srcs { if src.source == "myhostname" { - if hasDot { + if hostname == "" || hasDot { continue } - return hostLookupCgo + return fallbackOrder } if src.source == "files" || src.source == "dns" { if !src.standardCriteria() { - return hostLookupCgo // non-standard; let libc deal with it. + return fallbackOrder // non-standard; let libc deal with it. } if src.source == "files" { filesSource = true @@ -246,14 +247,14 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { continue } // Some source we don't know how to deal with. - return hostLookupCgo + return fallbackOrder } // We don't parse mdns.allow files. They're rare. If one // exists, it might list other TLDs (besides .local) or even // '*', so just let libc deal with it. if mdnsSource && c.hasMDNSAllow { - return hostLookupCgo + return fallbackOrder } // Cases where Go can handle it without cgo and C thread @@ -272,7 +273,7 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { } // Something weird. Let libc deal with it. - return hostLookupCgo + return fallbackOrder } // goDebugNetDNS parses the value of the GODEBUG "netdns" value. diff --git a/libgo/go/net/conf_netcgo.go b/libgo/go/net/conf_netcgo.go index b66bae3..abc33ce 100644 --- a/libgo/go/net/conf_netcgo.go +++ b/libgo/go/net/conf_netcgo.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/conf_test.go b/libgo/go/net/conf_test.go index 86904bf..ec8814b 100644 --- a/libgo/go/net/conf_test.go +++ b/libgo/go/net/conf_test.go @@ -32,7 +32,6 @@ func TestConfHostLookupOrder(t *testing.T) { tests := []struct { name string c *conf - goos string hostTests []nssHostTest }{ { @@ -48,6 +47,28 @@ func TestConfHostLookupOrder(t *testing.T) { }, }, { + name: "netgo_dns_before_files", + c: &conf{ + netGo: true, + nss: nssStr("hosts: dns files"), + resolv: defaultResolvConf, + }, + hostTests: []nssHostTest{ + {"x.com", hostLookupDNSFiles}, + }, + }, + { + name: "netgo_fallback_on_cgo", + c: &conf{ + netGo: true, + nss: nssStr("hosts: dns files something_custom"), + resolv: defaultResolvConf, + }, + hostTests: []nssHostTest{ + {"x.com", hostLookupFilesDNS}, + }, + }, + { name: "ubuntu_trusty_avahi", c: &conf{ nss: nssStr("hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4"), @@ -236,6 +257,7 @@ func TestConfHostLookupOrder(t *testing.T) { hostTests: []nssHostTest{ {"x.com", hostLookupFilesDNS}, {"somehostname", hostLookupCgo}, + {"", hostLookupFilesDNS}, // Issue 13623 }, }, { diff --git a/libgo/go/net/conn_test.go b/libgo/go/net/conn_test.go index 6995c11..16cf69e 100644 --- a/libgo/go/net/conn_test.go +++ b/libgo/go/net/conn_test.go @@ -1,4 +1,4 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -43,7 +43,7 @@ func TestConnAndListener(t *testing.T) { t.Fatal(err) } defer c.Close() - if c.LocalAddr().Network() != network || c.LocalAddr().Network() != network { + if c.LocalAddr().Network() != network || c.RemoteAddr().Network() != network { t.Fatalf("got %s->%s; want %s->%s", c.LocalAddr().Network(), c.RemoteAddr().Network(), network, network) } c.SetDeadline(time.Now().Add(someTimeout)) diff --git a/libgo/go/net/dial.go b/libgo/go/net/dial.go index 193776f..55edb43 100644 --- a/libgo/go/net/dial.go +++ b/libgo/go/net/dial.go @@ -1,11 +1,12 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( - "errors" + "context" + "internal/nettrace" "time" ) @@ -61,21 +62,34 @@ type Dialer struct { // Cancel is an optional channel whose closure indicates that // the dial should be canceled. Not all types of dials support // cancelation. + // + // Deprecated: Use DialContext instead. Cancel <-chan struct{} } -// Return either now+Timeout or Deadline, whichever comes first. -// Or zero, if neither is set. -func (d *Dialer) deadline(now time.Time) time.Time { - if d.Timeout == 0 { - return d.Deadline +func minNonzeroTime(a, b time.Time) time.Time { + if a.IsZero() { + return b } - timeoutDeadline := now.Add(d.Timeout) - if d.Deadline.IsZero() || timeoutDeadline.Before(d.Deadline) { - return timeoutDeadline - } else { - return d.Deadline + if b.IsZero() || a.Before(b) { + return a + } + return b +} + +// deadline returns the earliest of: +// - now+Timeout +// - d.Deadline +// - the context's deadline +// Or zero, if none of Timeout, Deadline, or context's deadline is set. +func (d *Dialer) deadline(ctx context.Context, now time.Time) (earliest time.Time) { + if d.Timeout != 0 { // including negative, for historical reasons + earliest = now.Add(d.Timeout) + } + if d, ok := ctx.Deadline(); ok { + earliest = minNonzeroTime(earliest, d) } + return minNonzeroTime(earliest, d.Deadline) } // partialDeadline returns the deadline to use for a single address, @@ -110,7 +124,7 @@ func (d *Dialer) fallbackDelay() time.Duration { } } -func parseNetwork(net string) (afnet string, proto int, err error) { +func parseNetwork(ctx context.Context, net string) (afnet string, proto int, err error) { i := last(net, ':') if i < 0 { // no colon switch net { @@ -129,7 +143,7 @@ func parseNetwork(net string) (afnet string, proto int, err error) { protostr := net[i+1:] proto, i, ok := dtoi(protostr, 0) if !ok || i != len(protostr) { - proto, err = lookupProtocol(protostr) + proto, err = lookupProtocol(ctx, protostr) if err != nil { return "", 0, err } @@ -139,8 +153,11 @@ func parseNetwork(net string) (afnet string, proto int, err error) { return "", 0, UnknownNetworkError(net) } -func resolveAddrList(op, net, addr string, deadline time.Time) (addrList, error) { - afnet, _, err := parseNetwork(net) +// resolverAddrList resolves addr using hint and returns a list of +// addresses. The result contains at least one address when error is +// nil. +func resolveAddrList(ctx context.Context, op, network, addr string, hint Addr) (addrList, error) { + afnet, _, err := parseNetwork(ctx, network) if err != nil { return nil, err } @@ -149,13 +166,64 @@ func resolveAddrList(op, net, addr string, deadline time.Time) (addrList, error) } switch afnet { case "unix", "unixgram", "unixpacket": + // TODO(bradfitz): push down context addr, err := ResolveUnixAddr(afnet, addr) if err != nil { return nil, err } + if op == "dial" && hint != nil && addr.Network() != hint.Network() { + return nil, &AddrError{Err: "mismatched local address type", Addr: hint.String()} + } return addrList{addr}, nil } - return internetAddrList(afnet, addr, deadline) + addrs, err := internetAddrList(ctx, afnet, addr) + if err != nil || op != "dial" || hint == nil { + return addrs, err + } + var ( + tcp *TCPAddr + udp *UDPAddr + ip *IPAddr + wildcard bool + ) + switch hint := hint.(type) { + case *TCPAddr: + tcp = hint + wildcard = tcp.isWildcard() + case *UDPAddr: + udp = hint + wildcard = udp.isWildcard() + case *IPAddr: + ip = hint + wildcard = ip.isWildcard() + } + naddrs := addrs[:0] + for _, addr := range addrs { + if addr.Network() != hint.Network() { + return nil, &AddrError{Err: "mismatched local address type", Addr: hint.String()} + } + switch addr := addr.(type) { + case *TCPAddr: + if !wildcard && !addr.isWildcard() && !addr.IP.matchAddrFamily(tcp.IP) { + continue + } + naddrs = append(naddrs, addr) + case *UDPAddr: + if !wildcard && !addr.isWildcard() && !addr.IP.matchAddrFamily(udp.IP) { + continue + } + naddrs = append(naddrs, addr) + case *IPAddr: + if !wildcard && !addr.isWildcard() && !addr.IP.matchAddrFamily(ip.IP) { + continue + } + naddrs = append(naddrs, addr) + } + } + if len(naddrs) == 0 { + return nil, errNoSuitableAddress + } + return naddrs, nil } // Dial connects to the address on the named network. @@ -173,8 +241,8 @@ func resolveAddrList(op, net, addr string, deadline time.Time) (addrList, error) // If the host is empty, as in ":80", the local system is assumed. // // Examples: -// Dial("tcp", "12.34.56.78:80") -// Dial("tcp", "google.com:http") +// Dial("tcp", "192.0.2.1:80") +// Dial("tcp", "golang.org:http") // Dial("tcp", "[2001:db8::1]:http") // Dial("tcp", "[fe80::1%lo0]:80") // Dial("tcp", ":80") @@ -184,8 +252,8 @@ func resolveAddrList(op, net, addr string, deadline time.Time) (addrList, error) // literal IP address. // // Examples: -// Dial("ip4:1", "127.0.0.1") -// Dial("ip6:ospf", "::1") +// Dial("ip4:1", "192.0.2.1") +// Dial("ip6:ipv6-icmp", "2001:db8::1") // // For Unix networks, the address must be a file system path. func Dial(network, address string) (Conn, error) { @@ -200,11 +268,10 @@ func DialTimeout(network, address string, timeout time.Duration) (Conn, error) { return d.Dial(network, address) } -// dialContext holds common state for all dial operations. -type dialContext struct { +// dialParam contains a Dial's parameters and configuration. +type dialParam struct { Dialer network, address string - finalDeadline time.Time } // Dial connects to the address on the named network. @@ -212,17 +279,62 @@ type dialContext struct { // See func Dial for a description of the network and address // parameters. func (d *Dialer) Dial(network, address string) (Conn, error) { - finalDeadline := d.deadline(time.Now()) - addrs, err := resolveAddrList("dial", network, address, finalDeadline) + return d.DialContext(context.Background(), network, address) +} + +// DialContext connects to the address on the named network using +// the provided context. +// +// The provided Context must be non-nil. If the context expires before +// the connection is complete, an error is returned. Once successfully +// connected, any expiration of the context will not affect the +// connection. +// +// See func Dial for a description of the network and address +// parameters. +func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn, error) { + if ctx == nil { + panic("nil context") + } + deadline := d.deadline(ctx, time.Now()) + if !deadline.IsZero() { + if d, ok := ctx.Deadline(); !ok || deadline.Before(d) { + subCtx, cancel := context.WithDeadline(ctx, deadline) + defer cancel() + ctx = subCtx + } + } + if oldCancel := d.Cancel; oldCancel != nil { + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + select { + case <-oldCancel: + cancel() + case <-subCtx.Done(): + } + }() + ctx = subCtx + } + + // Shadow the nettrace (if any) during resolve so Connect events don't fire for DNS lookups. + resolveCtx := ctx + if trace, _ := ctx.Value(nettrace.TraceKey{}).(*nettrace.Trace); trace != nil { + shadow := *trace + shadow.ConnectStart = nil + shadow.ConnectDone = nil + resolveCtx = context.WithValue(resolveCtx, nettrace.TraceKey{}, &shadow) + } + + addrs, err := resolveAddrList(resolveCtx, "dial", network, address, d.LocalAddr) if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: nil, Addr: nil, Err: err} } - ctx := &dialContext{ - Dialer: *d, - network: network, - address: address, - finalDeadline: finalDeadline, + dp := &dialParam{ + Dialer: *d, + network: network, + address: address, } var primaries, fallbacks addrList @@ -233,116 +345,128 @@ func (d *Dialer) Dial(network, address string) (Conn, error) { } var c Conn - if len(fallbacks) == 0 { - // dialParallel can accept an empty fallbacks list, - // but this shortcut avoids the goroutine/channel overhead. - c, err = dialSerial(ctx, primaries, nil) + if len(fallbacks) > 0 { + c, err = dialParallel(ctx, dp, primaries, fallbacks) } else { - c, err = dialParallel(ctx, primaries, fallbacks) + c, err = dialSerial(ctx, dp, primaries) + } + if err != nil { + return nil, err } - if d.KeepAlive > 0 && err == nil { - if tc, ok := c.(*TCPConn); ok { - setKeepAlive(tc.fd, true) - setKeepAlivePeriod(tc.fd, d.KeepAlive) - testHookSetKeepAlive() - } + if tc, ok := c.(*TCPConn); ok && d.KeepAlive > 0 { + setKeepAlive(tc.fd, true) + setKeepAlivePeriod(tc.fd, d.KeepAlive) + testHookSetKeepAlive() } - return c, err + return c, nil } // dialParallel races two copies of dialSerial, giving the first a // head start. It returns the first established connection and // closes the others. Otherwise it returns an error from the first // primary address. -func dialParallel(ctx *dialContext, primaries, fallbacks addrList) (Conn, error) { - results := make(chan dialResult) // unbuffered, so dialSerialAsync can detect race loss & cleanup - cancel := make(chan struct{}) - defer close(cancel) - - // Spawn the primary racer. - go dialSerialAsync(ctx, primaries, nil, cancel, results) - - // Spawn the fallback racer. - fallbackTimer := time.NewTimer(ctx.fallbackDelay()) - go dialSerialAsync(ctx, fallbacks, fallbackTimer, cancel, results) - - var primaryErr error - for nracers := 2; nracers > 0; nracers-- { - res := <-results - // If we're still waiting for a connection, then hasten the delay. - // Otherwise, disable the Timer and let cancel take over. - if fallbackTimer.Stop() && res.error != nil { - fallbackTimer.Reset(0) - } - if res.error == nil { - return res.Conn, nil - } - if res.primary { - primaryErr = res.error - } +func dialParallel(ctx context.Context, dp *dialParam, primaries, fallbacks addrList) (Conn, error) { + if len(fallbacks) == 0 { + return dialSerial(ctx, dp, primaries) } - return nil, primaryErr -} -type dialResult struct { - Conn - error - primary bool -} + returned := make(chan struct{}) + defer close(returned) + + type dialResult struct { + Conn + error + primary bool + done bool + } + results := make(chan dialResult) // unbuffered -// dialSerialAsync runs dialSerial after some delay, and returns the -// resulting connection through a channel. When racing two connections, -// the primary goroutine uses a nil timer to omit the delay. -func dialSerialAsync(ctx *dialContext, ras addrList, timer *time.Timer, cancel <-chan struct{}, results chan<- dialResult) { - if timer != nil { - // We're in the fallback goroutine; sleep before connecting. + startRacer := func(ctx context.Context, primary bool) { + ras := primaries + if !primary { + ras = fallbacks + } + c, err := dialSerial(ctx, dp, ras) select { - case <-timer.C: - case <-cancel: - return + case results <- dialResult{Conn: c, error: err, primary: primary, done: true}: + case <-returned: + if c != nil { + c.Close() + } } } - c, err := dialSerial(ctx, ras, cancel) - select { - case results <- dialResult{c, err, timer == nil}: - // We won the race. - case <-cancel: - // The other goroutine won the race. - if c != nil { - c.Close() + + var primary, fallback dialResult + + // Start the main racer. + primaryCtx, primaryCancel := context.WithCancel(ctx) + defer primaryCancel() + go startRacer(primaryCtx, true) + + // Start the timer for the fallback racer. + fallbackTimer := time.NewTimer(dp.fallbackDelay()) + defer fallbackTimer.Stop() + + for { + select { + case <-fallbackTimer.C: + fallbackCtx, fallbackCancel := context.WithCancel(ctx) + defer fallbackCancel() + go startRacer(fallbackCtx, false) + + case res := <-results: + if res.error == nil { + return res.Conn, nil + } + if res.primary { + primary = res + } else { + fallback = res + } + if primary.done && fallback.done { + return nil, primary.error + } + if res.primary && fallbackTimer.Stop() { + // If we were able to stop the timer, that means it + // was running (hadn't yet started the fallback), but + // we just got an error on the primary path, so start + // the fallback immediately (in 0 nanoseconds). + fallbackTimer.Reset(0) + } } } } // dialSerial connects to a list of addresses in sequence, returning // either the first successful connection, or the first error. -func dialSerial(ctx *dialContext, ras addrList, cancel <-chan struct{}) (Conn, error) { +func dialSerial(ctx context.Context, dp *dialParam, ras addrList) (Conn, error) { var firstErr error // The error from the first address is most relevant. for i, ra := range ras { select { - case <-cancel: - return nil, &OpError{Op: "dial", Net: ctx.network, Source: ctx.LocalAddr, Addr: ra, Err: errCanceled} + case <-ctx.Done(): + return nil, &OpError{Op: "dial", Net: dp.network, Source: dp.LocalAddr, Addr: ra, Err: mapErr(ctx.Err())} default: } - partialDeadline, err := partialDeadline(time.Now(), ctx.finalDeadline, len(ras)-i) + deadline, _ := ctx.Deadline() + partialDeadline, err := partialDeadline(time.Now(), deadline, len(ras)-i) if err != nil { // Ran out of time. if firstErr == nil { - firstErr = &OpError{Op: "dial", Net: ctx.network, Source: ctx.LocalAddr, Addr: ra, Err: err} + firstErr = &OpError{Op: "dial", Net: dp.network, Source: dp.LocalAddr, Addr: ra, Err: err} } break } - - // dialTCP does not support cancelation (see golang.org/issue/11225), - // so if cancel fires, we'll continue trying to connect until the next - // timeout, or return a spurious connection for the caller to close. - dialer := func(d time.Time) (Conn, error) { - return dialSingle(ctx, ra, d) + dialCtx := ctx + if partialDeadline.Before(deadline) { + var cancel context.CancelFunc + dialCtx, cancel = context.WithDeadline(ctx, partialDeadline) + defer cancel() } - c, err := dial(ctx.network, ra, dialer, partialDeadline) + + c, err := dialSingle(dialCtx, dp, ra) if err == nil { return c, nil } @@ -352,37 +476,43 @@ func dialSerial(ctx *dialContext, ras addrList, cancel <-chan struct{}) (Conn, e } if firstErr == nil { - firstErr = &OpError{Op: "dial", Net: ctx.network, Source: nil, Addr: nil, Err: errMissingAddress} + firstErr = &OpError{Op: "dial", Net: dp.network, Source: nil, Addr: nil, Err: errMissingAddress} } return nil, firstErr } // dialSingle attempts to establish and returns a single connection to -// the destination address. This must be called through the OS-specific -// dial function, because some OSes don't implement the deadline feature. -func dialSingle(ctx *dialContext, ra Addr, deadline time.Time) (c Conn, err error) { - la := ctx.LocalAddr - if la != nil && la.Network() != ra.Network() { - return nil, &OpError{Op: "dial", Net: ctx.network, Source: la, Addr: ra, Err: errors.New("mismatched local address type " + la.Network())} +// the destination address. +func dialSingle(ctx context.Context, dp *dialParam, ra Addr) (c Conn, err error) { + trace, _ := ctx.Value(nettrace.TraceKey{}).(*nettrace.Trace) + if trace != nil { + raStr := ra.String() + if trace.ConnectStart != nil { + trace.ConnectStart(dp.network, raStr) + } + if trace.ConnectDone != nil { + defer func() { trace.ConnectDone(dp.network, raStr, err) }() + } } + la := dp.LocalAddr switch ra := ra.(type) { case *TCPAddr: la, _ := la.(*TCPAddr) - c, err = testHookDialTCP(ctx.network, la, ra, deadline, ctx.Cancel) + c, err = dialTCP(ctx, dp.network, la, ra) case *UDPAddr: la, _ := la.(*UDPAddr) - c, err = dialUDP(ctx.network, la, ra, deadline) + c, err = dialUDP(ctx, dp.network, la, ra) case *IPAddr: la, _ := la.(*IPAddr) - c, err = dialIP(ctx.network, la, ra, deadline) + c, err = dialIP(ctx, dp.network, la, ra) case *UnixAddr: la, _ := la.(*UnixAddr) - c, err = dialUnix(ctx.network, la, ra, deadline) + c, err = dialUnix(ctx, dp.network, la, ra) default: - return nil, &OpError{Op: "dial", Net: ctx.network, Source: la, Addr: ra, Err: &AddrError{Err: "unexpected address type", Addr: ctx.address}} + return nil, &OpError{Op: "dial", Net: dp.network, Source: la, Addr: ra, Err: &AddrError{Err: "unexpected address type", Addr: dp.address}} } if err != nil { - return nil, err // c is non-nil interface containing nil pointer + return nil, &OpError{Op: "dial", Net: dp.network, Source: la, Addr: ra, Err: err} // c is non-nil interface containing nil pointer } return c, nil } @@ -395,7 +525,7 @@ func dialSingle(ctx *dialContext, ra Addr, deadline time.Time) (c Conn, err erro // instead of just the interface with the given host address. // See Dial for more details about address syntax. func Listen(net, laddr string) (Listener, error) { - addrs, err := resolveAddrList("listen", net, laddr, noDeadline) + addrs, err := resolveAddrList(context.Background(), "listen", net, laddr, nil) if err != nil { return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: err} } @@ -422,7 +552,7 @@ func Listen(net, laddr string) (Listener, error) { // instead of just the interface with the given host address. // See Dial for the syntax of laddr. func ListenPacket(net, laddr string) (PacketConn, error) { - addrs, err := resolveAddrList("listen", net, laddr, noDeadline) + addrs, err := resolveAddrList(context.Background(), "listen", net, laddr, nil) if err != nil { return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: err} } diff --git a/libgo/go/net/dial_gen.go b/libgo/go/net/dial_gen.go deleted file mode 100644 index a628f71..0000000 --- a/libgo/go/net/dial_gen.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows plan9 - -package net - -import "time" - -// dialChannel is the simple pure-Go implementation of dial, still -// used on operating systems where the deadline hasn't been pushed -// down into the pollserver. (Plan 9 and some old versions of Windows) -func dialChannel(net string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { - if deadline.IsZero() { - return dialer(noDeadline) - } - timeout := deadline.Sub(time.Now()) - if timeout <= 0 { - return nil, &OpError{Op: "dial", Net: net, Source: nil, Addr: ra, Err: errTimeout} - } - t := time.NewTimer(timeout) - defer t.Stop() - type racer struct { - Conn - error - } - ch := make(chan racer, 1) - go func() { - testHookDialChannel() - c, err := dialer(noDeadline) - ch <- racer{c, err} - }() - select { - case <-t.C: - return nil, &OpError{Op: "dial", Net: net, Source: nil, Addr: ra, Err: errTimeout} - case racer := <-ch: - return racer.Conn, racer.error - } -} diff --git a/libgo/go/net/dial_test.go b/libgo/go/net/dial_test.go index 2311b10..8b21e6b 100644 --- a/libgo/go/net/dial_test.go +++ b/libgo/go/net/dial_test.go @@ -5,6 +5,8 @@ package net import ( + "bufio" + "context" "internal/testenv" "io" "net/internal/socktest" @@ -23,13 +25,12 @@ var prohibitionaryDialArgTests = []struct { } func TestProhibitionaryDialArg(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + switch runtime.GOOS { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } if !supportsIPv4map { t.Skip("mapping ipv4 address inside ipv6 address not supported") } @@ -54,56 +55,12 @@ func TestProhibitionaryDialArg(t *testing.T) { } } -func TestSelfConnect(t *testing.T) { - if runtime.GOOS == "windows" { - // TODO(brainman): do not know why it hangs. - t.Skip("known-broken test on windows") - } - - // Test that Dial does not honor self-connects. - // See the comment in DialTCP. - - // Find a port that would be used as a local address. - l, err := Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - c, err := Dial("tcp", l.Addr().String()) - if err != nil { - t.Fatal(err) - } - addr := c.LocalAddr().String() - c.Close() - l.Close() - - // Try to connect to that address repeatedly. - n := 100000 - if testing.Short() { - n = 1000 - } - switch runtime.GOOS { - case "darwin", "dragonfly", "freebsd", "netbsd", "openbsd", "plan9", "solaris", "windows": - // Non-Linux systems take a long time to figure - // out that there is nothing listening on localhost. - n = 100 - } - for i := 0; i < n; i++ { - c, err := DialTimeout("tcp", addr, time.Millisecond) - if err == nil { - if c.LocalAddr().String() == addr { - t.Errorf("#%d: Dial %q self-connect", i, addr) - } else { - t.Logf("#%d: Dial %q succeeded - possibly racing with other listener", i, addr) - } - c.Close() - } - } -} - func TestDialTimeoutFDLeak(t *testing.T) { switch runtime.GOOS { case "plan9": t.Skipf("%s does not have full support of socktest", runtime.GOOS) + case "openbsd": + testenv.SkipFlaky(t, 15157) } const T = 100 * time.Millisecond @@ -130,17 +87,14 @@ func TestDialTimeoutFDLeak(t *testing.T) { // socktest.Switch. // It may happen when the Dial call bumps against TCP // simultaneous open. See selfConnect in tcpsock_posix.go. - defer func() { - sw.Set(socktest.FilterClose, nil) - forceCloseSockets() - }() + defer func() { sw.Set(socktest.FilterClose, nil) }() var mu sync.Mutex var attempts int sw.Set(socktest.FilterClose, func(so *socktest.Status) (socktest.AfterFilter, error) { mu.Lock() attempts++ mu.Unlock() - return nil, errTimedout + return nil, nil }) const N = 100 @@ -176,6 +130,12 @@ func TestDialerDualStackFDLeak(t *testing.T) { t.Skip("both IPv4 and IPv6 are required") } + closedPortDelay, expectClosedPortDelay := dialClosedPort() + if closedPortDelay > expectClosedPortDelay { + t.Errorf("got %v; want <= %v", closedPortDelay, expectClosedPortDelay) + } + + before := sw.Sockets() origTestHookLookupIP := testHookLookupIP defer func() { testHookLookupIP = origTestHookLookupIP }() testHookLookupIP = lookupLocalhost @@ -188,24 +148,19 @@ func TestDialerDualStackFDLeak(t *testing.T) { c.Close() } } - dss, err := newDualStackServer([]streamListener{ - {network: "tcp4", address: "127.0.0.1"}, - {network: "tcp6", address: "::1"}, - }) + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } - defer dss.teardown() if err := dss.buildup(handler); err != nil { + dss.teardown() t.Fatal(err) } - before := sw.Sockets() - const T = 100 * time.Millisecond const N = 10 var wg sync.WaitGroup wg.Add(N) - d := &Dialer{DualStack: true, Timeout: T} + d := &Dialer{DualStack: true, Timeout: 100*time.Millisecond + closedPortDelay} for i := 0; i < N; i++ { go func() { defer wg.Done() @@ -218,7 +173,7 @@ func TestDialerDualStackFDLeak(t *testing.T) { }() } wg.Wait() - time.Sleep(2 * T) // wait for the dial racers to stop + dss.teardown() after := sw.Sockets() if len(after) != len(before) { t.Errorf("got %d; want %d", len(after), len(before)) @@ -229,18 +184,18 @@ func TestDialerDualStackFDLeak(t *testing.T) { // expected to hang until the timeout elapses. These addresses are reserved // for benchmarking by RFC 6890. const ( - slowDst4 = "192.18.0.254" - slowDst6 = "2001:2::254" - slowTimeout = 1 * time.Second + slowDst4 = "198.18.0.254" + slowDst6 = "2001:2::254" ) // In some environments, the slow IPs may be explicitly unreachable, and fail // more quickly than expected. This test hook prevents dialTCP from returning // before the deadline. -func slowDialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time, cancel <-chan struct{}) (*TCPConn, error) { - c, err := dialTCP(net, laddr, raddr, deadline, cancel) +func slowDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + c, err := doDialTCP(ctx, net, laddr, raddr) if ParseIP(slowDst4).Equal(raddr.IP) || ParseIP(slowDst6).Equal(raddr.IP) { - time.Sleep(deadline.Sub(time.Now())) + // Wait for the deadline, or indefinitely if none exists. + <-ctx.Done() } return c, err } @@ -278,9 +233,8 @@ func dialClosedPort() (actual, expected time.Duration) { } func TestDialParallel(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv4 || !supportsIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -294,7 +248,7 @@ func TestDialParallel(t *testing.T) { const fallbackDelay = 200 * time.Millisecond // Some cases will run quickly when "connection refused" is fast, - // or trigger the fallbackDelay on Windows. This value holds the + // or trigger the fallbackDelay on Windows. This value holds the // lesser of the two delays. var closedPortOrFallbackDelay time.Duration if closedPortDelay < fallbackDelay { @@ -369,10 +323,7 @@ func TestDialParallel(t *testing.T) { } for i, tt := range testCases { - dss, err := newDualStackServer([]streamListener{ - {network: "tcp4", address: "127.0.0.1"}, - {network: "tcp6", address: "::1"}, - }) + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } @@ -389,17 +340,15 @@ func TestDialParallel(t *testing.T) { fallbacks := makeAddrs(tt.fallbacks, dss.port) d := Dialer{ FallbackDelay: fallbackDelay, - Timeout: slowTimeout, - } - ctx := &dialContext{ - Dialer: d, - network: "tcp", - address: "?", - finalDeadline: d.deadline(time.Now()), } startTime := time.Now() - c, err := dialParallel(ctx, primaries, fallbacks) - elapsed := time.Now().Sub(startTime) + dp := &dialParam{ + Dialer: d, + network: "tcp", + address: "?", + } + c, err := dialParallel(context.Background(), dp, primaries, fallbacks) + elapsed := time.Since(startTime) if c != nil { c.Close() @@ -418,12 +367,30 @@ func TestDialParallel(t *testing.T) { } else if !(elapsed <= expectElapsedMax) { t.Errorf("#%d: got %v; want <= %v", i, elapsed, expectElapsedMax) } + + // Repeat each case, ensuring that it can be canceled quickly. + ctx, cancel := context.WithCancel(context.Background()) + var wg sync.WaitGroup + wg.Add(1) + go func() { + time.Sleep(5 * time.Millisecond) + cancel() + wg.Done() + }() + startTime = time.Now() + c, err = dialParallel(ctx, dp, primaries, fallbacks) + if c != nil { + c.Close() + } + elapsed = time.Now().Sub(startTime) + if elapsed > 100*time.Millisecond { + t.Errorf("#%d (cancel): got %v; want <= 100ms", i, elapsed) + } + wg.Wait() } - // Wait for any slowDst4/slowDst6 connections to timeout. - time.Sleep(slowTimeout * 3 / 2) } -func lookupSlowFast(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { +func lookupSlowFast(ctx context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { switch host { case "slow6loopback4": // Returns a slow IPv6 address, and a local IPv4 address. @@ -432,14 +399,13 @@ func lookupSlowFast(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, e {IP: ParseIP("127.0.0.1")}, }, nil default: - return fn(host) + return fn(ctx, host) } } func TestDialerFallbackDelay(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv4 || !supportsIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -463,8 +429,6 @@ func TestDialerFallbackDelay(t *testing.T) { {true, 200 * time.Millisecond, 200 * time.Millisecond}, // The default is 300ms. {true, 0, 300 * time.Millisecond}, - // This case is last, in order to wait for hanging slowDst6 connections. - {false, 0, slowTimeout}, } handler := func(dss *dualStackServer, ln Listener) { @@ -476,9 +440,7 @@ func TestDialerFallbackDelay(t *testing.T) { c.Close() } } - dss, err := newDualStackServer([]streamListener{ - {network: "tcp", address: "127.0.0.1"}, - }) + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } @@ -488,7 +450,7 @@ func TestDialerFallbackDelay(t *testing.T) { } for i, tt := range testCases { - d := &Dialer{DualStack: tt.dualstack, FallbackDelay: tt.delay, Timeout: slowTimeout} + d := &Dialer{DualStack: tt.dualstack, FallbackDelay: tt.delay} startTime := time.Now() c, err := d.Dial("tcp", JoinHostPort("slow6loopback4", dss.port)) @@ -509,46 +471,78 @@ func TestDialerFallbackDelay(t *testing.T) { } } -func TestDialSerialAsyncSpuriousConnection(t *testing.T) { - if runtime.GOOS == "plan9" { - t.Skip("skipping on plan9; no deadline support, golang.org/issue/11932") +func TestDialParallelSpuriousConnection(t *testing.T) { + if !supportsIPv4 || !supportsIPv6 { + t.Skip("both IPv4 and IPv6 are required") } - ln, err := newLocalListener("tcp") + + var wg sync.WaitGroup + wg.Add(2) + handler := func(dss *dualStackServer, ln Listener) { + // Accept one connection per address. + c, err := ln.Accept() + if err != nil { + t.Fatal(err) + } + // The client should close itself, without sending data. + c.SetReadDeadline(time.Now().Add(1 * time.Second)) + var b [1]byte + if _, err := c.Read(b[:]); err != io.EOF { + t.Errorf("got %v; want %v", err, io.EOF) + } + c.Close() + wg.Done() + } + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } - defer ln.Close() + defer dss.teardown() + if err := dss.buildup(handler); err != nil { + t.Fatal(err) + } + + const fallbackDelay = 100 * time.Millisecond + + origTestHookDialTCP := testHookDialTCP + defer func() { testHookDialTCP = origTestHookDialTCP }() + testHookDialTCP = func(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + // Sleep long enough for Happy Eyeballs to kick in, and inhibit cancelation. + // This forces dialParallel to juggle two successful connections. + time.Sleep(fallbackDelay * 2) - d := Dialer{} - ctx := &dialContext{ - Dialer: d, - network: "tcp", - address: "?", - finalDeadline: d.deadline(time.Now()), + // Now ignore the provided context (which will be canceled) and use a + // different one to make sure this completes with a valid connection, + // which we hope to be closed below: + return doDialTCP(context.Background(), net, laddr, raddr) } - results := make(chan dialResult) - cancel := make(chan struct{}) + d := Dialer{ + FallbackDelay: fallbackDelay, + } + dp := &dialParam{ + Dialer: d, + network: "tcp", + address: "?", + } - // Spawn a connection in the background. - go dialSerialAsync(ctx, addrList{ln.Addr()}, nil, cancel, results) + makeAddr := func(ip string) addrList { + addr, err := ResolveTCPAddr("tcp", JoinHostPort(ip, dss.port)) + if err != nil { + t.Fatal(err) + } + return addrList{addr} + } - // Receive it at the server. - c, err := ln.Accept() + // dialParallel returns one connection (and closes the other.) + c, err := dialParallel(context.Background(), dp, makeAddr("127.0.0.1"), makeAddr("::1")) if err != nil { t.Fatal(err) } - defer c.Close() - - // Tell dialSerialAsync that someone else won the race. - close(cancel) + c.Close() - // The connection should close itself, without sending data. - c.SetReadDeadline(time.Now().Add(1 * time.Second)) - var b [1]byte - if _, err := c.Read(b[:]); err != io.EOF { - t.Errorf("got %v; want %v", err, io.EOF) - } + // The server should've seen both connections. + wg.Wait() } func TestDialerPartialDeadline(t *testing.T) { @@ -587,44 +581,125 @@ func TestDialerPartialDeadline(t *testing.T) { } func TestDialerLocalAddr(t *testing.T) { - ch := make(chan error, 1) - handler := func(ls *localServer, ln Listener) { - c, err := ln.Accept() - if err != nil { - ch <- err - return - } - defer c.Close() - ch <- nil - } - ls, err := newLocalServer("tcp") - if err != nil { - t.Fatal(err) + if !supportsIPv4 || !supportsIPv6 { + t.Skip("both IPv4 and IPv6 are required") } - defer ls.teardown() - if err := ls.buildup(handler); err != nil { - t.Fatal(err) + + type test struct { + network, raddr string + laddr Addr + error + } + var tests = []test{ + {"tcp4", "127.0.0.1", nil, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("0.0.0.0")}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("0.0.0.0").To4()}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("::")}, &AddrError{Err: "some error"}}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("127.0.0.1").To4()}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("127.0.0.1").To16()}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: IPv6loopback}, errNoSuitableAddress}, + {"tcp4", "127.0.0.1", &UDPAddr{}, &AddrError{Err: "some error"}}, + {"tcp4", "127.0.0.1", &UnixAddr{}, &AddrError{Err: "some error"}}, + + {"tcp6", "::1", nil, nil}, + {"tcp6", "::1", &TCPAddr{}, nil}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("0.0.0.0")}, nil}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("0.0.0.0").To4()}, nil}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("::")}, nil}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("127.0.0.1").To4()}, errNoSuitableAddress}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("127.0.0.1").To16()}, errNoSuitableAddress}, + {"tcp6", "::1", &TCPAddr{IP: IPv6loopback}, nil}, + {"tcp6", "::1", &UDPAddr{}, &AddrError{Err: "some error"}}, + {"tcp6", "::1", &UnixAddr{}, &AddrError{Err: "some error"}}, + + {"tcp", "127.0.0.1", nil, nil}, + {"tcp", "127.0.0.1", &TCPAddr{}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("0.0.0.0")}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("0.0.0.0").To4()}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("127.0.0.1").To4()}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("127.0.0.1").To16()}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: IPv6loopback}, errNoSuitableAddress}, + {"tcp", "127.0.0.1", &UDPAddr{}, &AddrError{Err: "some error"}}, + {"tcp", "127.0.0.1", &UnixAddr{}, &AddrError{Err: "some error"}}, + + {"tcp", "::1", nil, nil}, + {"tcp", "::1", &TCPAddr{}, nil}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("0.0.0.0")}, nil}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("0.0.0.0").To4()}, nil}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("::")}, nil}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("127.0.0.1").To4()}, errNoSuitableAddress}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("127.0.0.1").To16()}, errNoSuitableAddress}, + {"tcp", "::1", &TCPAddr{IP: IPv6loopback}, nil}, + {"tcp", "::1", &UDPAddr{}, &AddrError{Err: "some error"}}, + {"tcp", "::1", &UnixAddr{}, &AddrError{Err: "some error"}}, + } + + if supportsIPv4map { + tests = append(tests, test{ + "tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("::")}, nil, + }) + } else { + tests = append(tests, test{ + "tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("::")}, &AddrError{Err: "some error"}, + }) } - laddr, err := ResolveTCPAddr(ls.Listener.Addr().Network(), ls.Listener.Addr().String()) - if err != nil { - t.Fatal(err) + origTestHookLookupIP := testHookLookupIP + defer func() { testHookLookupIP = origTestHookLookupIP }() + testHookLookupIP = lookupLocalhost + handler := func(ls *localServer, ln Listener) { + for { + c, err := ln.Accept() + if err != nil { + return + } + c.Close() + } } - laddr.Port = 0 - d := &Dialer{LocalAddr: laddr} - c, err := d.Dial(ls.Listener.Addr().Network(), ls.Addr().String()) - if err != nil { - t.Fatal(err) + var err error + var lss [2]*localServer + for i, network := range []string{"tcp4", "tcp6"} { + lss[i], err = newLocalServer(network) + if err != nil { + t.Fatal(err) + } + defer lss[i].teardown() + if err := lss[i].buildup(handler); err != nil { + t.Fatal(err) + } } - defer c.Close() - c.Read(make([]byte, 1)) - err = <-ch - if err != nil { - t.Error(err) + + for _, tt := range tests { + d := &Dialer{LocalAddr: tt.laddr} + var addr string + ip := ParseIP(tt.raddr) + if ip.To4() != nil { + addr = lss[0].Listener.Addr().String() + } + if ip.To16() != nil && ip.To4() == nil { + addr = lss[1].Listener.Addr().String() + } + c, err := d.Dial(tt.network, addr) + if err == nil && tt.error != nil || err != nil && tt.error == nil { + t.Errorf("%s %v->%s: got %v; want %v", tt.network, tt.laddr, tt.raddr, err, tt.error) + } + if err != nil { + if perr := parseDialError(err); perr != nil { + t.Error(perr) + } + continue + } + c.Close() } } func TestDialerDualStack(t *testing.T) { + // This test is known to be flaky. Don't frighten regular + // users about it; only fail on the build dashboard. + if testenv.Builder() == "" { + testenv.SkipFlaky(t, 13324) + } if !supportsIPv4 || !supportsIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -649,10 +724,7 @@ func TestDialerDualStack(t *testing.T) { var timeout = 150*time.Millisecond + closedPortDelay for _, dualstack := range []bool{false, true} { - dss, err := newDualStackServer([]streamListener{ - {network: "tcp4", address: "127.0.0.1"}, - {network: "tcp6", address: "::1"}, - }) + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } @@ -677,7 +749,6 @@ func TestDialerDualStack(t *testing.T) { c.Close() } } - time.Sleep(timeout * 3 / 2) // wait for the dial racers to stop } func TestDialerKeepAlive(t *testing.T) { @@ -719,14 +790,16 @@ func TestDialerKeepAlive(t *testing.T) { } func TestDialCancel(t *testing.T) { - if runtime.GOOS == "plan9" || runtime.GOOS == "nacl" { - // plan9 is not implemented and nacl doesn't have - // external network access. - t.Skipf("skipping on %s", runtime.GOOS) + switch testenv.Builder() { + case "linux-arm64-buildlet": + t.Skip("skipping on linux-arm64-buildlet; incompatible network config? issue 15191") + case "": + testenv.MustHaveExternalNetwork(t) } - onGoBuildFarm := testenv.Builder() != "" - if testing.Short() && !onGoBuildFarm { - t.Skip("skipping in short mode") + + if runtime.GOOS == "nacl" { + // nacl doesn't have external network access. + t.Skipf("skipping on %s", runtime.GOOS) } blackholeIPPort := JoinHostPort(slowDst4, "1234") @@ -781,3 +854,84 @@ func TestDialCancel(t *testing.T) { } } } + +func TestCancelAfterDial(t *testing.T) { + if testing.Short() { + t.Skip("avoiding time.Sleep") + } + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + wg.Add(1) + defer func() { + ln.Close() + wg.Wait() + }() + + // Echo back the first line of each incoming connection. + go func() { + for { + c, err := ln.Accept() + if err != nil { + break + } + rb := bufio.NewReader(c) + line, err := rb.ReadString('\n') + if err != nil { + t.Error(err) + c.Close() + continue + } + if _, err := c.Write([]byte(line)); err != nil { + t.Error(err) + } + c.Close() + } + wg.Done() + }() + + try := func() { + cancel := make(chan struct{}) + d := &Dialer{Cancel: cancel} + c, err := d.Dial("tcp", ln.Addr().String()) + + // Immediately after dialing, request cancelation and sleep. + // Before Issue 15078 was fixed, this would cause subsequent operations + // to fail with an i/o timeout roughly 50% of the time. + close(cancel) + time.Sleep(10 * time.Millisecond) + + if err != nil { + t.Fatal(err) + } + defer c.Close() + + // Send some data to confirm that the connection is still alive. + const message = "echo!\n" + if _, err := c.Write([]byte(message)); err != nil { + t.Fatal(err) + } + + // The server should echo the line, and close the connection. + rb := bufio.NewReader(c) + line, err := rb.ReadString('\n') + if err != nil { + t.Fatal(err) + } + if line != message { + t.Errorf("got %q; want %q", line, message) + } + if _, err := rb.ReadByte(); err != io.EOF { + t.Errorf("got %v; want %v", err, io.EOF) + } + } + + // This bug manifested about 50% of the time, so try it a few times. + for i := 0; i < 10; i++ { + try() + } +} diff --git a/libgo/go/net/dnsclient.go b/libgo/go/net/dnsclient.go index 5dc2a03..f1835b8 100644 --- a/libgo/go/net/dnsclient.go +++ b/libgo/go/net/dnsclient.go @@ -45,7 +45,7 @@ func answer(name, server string, dns *dnsMsg, qtype uint16) (cname string, addrs } if dns.rcode != dnsRcodeSuccess { // None of the error codes make sense - // for the query we sent. If we didn't get + // for the query we sent. If we didn't get // a name error and we didn't get success, // the server is behaving incorrectly or // having temporary trouble. @@ -161,7 +161,7 @@ func isDomainName(s string) bool { return ok } -// absDomainName returns an absoulte domain name which ends with a +// absDomainName returns an absolute domain name which ends with a // trailing dot to match pure Go reverse resolver and all other lookup // routines. // See golang.org/issue/12189. diff --git a/libgo/go/net/dnsclient_unix.go b/libgo/go/net/dnsclient_unix.go index 17188f0..8f2dff4 100644 --- a/libgo/go/net/dnsclient_unix.go +++ b/libgo/go/net/dnsclient_unix.go @@ -16,6 +16,7 @@ package net import ( + "context" "errors" "io" "math/rand" @@ -26,10 +27,10 @@ import ( // A dnsDialer provides dialing suitable for DNS queries. type dnsDialer interface { - dialDNS(string, string) (dnsConn, error) + dialDNS(ctx context.Context, network, addr string) (dnsConn, error) } -var testHookDNSDialer = func(d time.Duration) dnsDialer { return &Dialer{Timeout: d} } +var testHookDNSDialer = func() dnsDialer { return &Dialer{} } // A dnsConn represents a DNS transport endpoint. type dnsConn interface { @@ -37,46 +38,67 @@ type dnsConn interface { SetDeadline(time.Time) error - // readDNSResponse reads a DNS response message from the DNS - // transport endpoint and returns the received DNS response - // message. - readDNSResponse() (*dnsMsg, error) + // dnsRoundTrip executes a single DNS transaction, returning a + // DNS response message for the provided DNS query message. + dnsRoundTrip(query *dnsMsg) (*dnsMsg, error) +} - // writeDNSQuery writes a DNS query message to the DNS - // connection endpoint. - writeDNSQuery(*dnsMsg) error +func (c *UDPConn) dnsRoundTrip(query *dnsMsg) (*dnsMsg, error) { + return dnsRoundTripUDP(c, query) } -func (c *UDPConn) readDNSResponse() (*dnsMsg, error) { - b := make([]byte, 512) // see RFC 1035 - n, err := c.Read(b) - if err != nil { +// dnsRoundTripUDP implements the dnsRoundTrip interface for RFC 1035's +// "UDP usage" transport mechanism. c should be a packet-oriented connection, +// such as a *UDPConn. +func dnsRoundTripUDP(c io.ReadWriter, query *dnsMsg) (*dnsMsg, error) { + b, ok := query.Pack() + if !ok { + return nil, errors.New("cannot marshal DNS message") + } + if _, err := c.Write(b); err != nil { return nil, err } - msg := &dnsMsg{} - if !msg.Unpack(b[:n]) { - return nil, errors.New("cannot unmarshal DNS message") + + b = make([]byte, 512) // see RFC 1035 + for { + n, err := c.Read(b) + if err != nil { + return nil, err + } + resp := &dnsMsg{} + if !resp.Unpack(b[:n]) || !resp.IsResponseTo(query) { + // Ignore invalid responses as they may be malicious + // forgery attempts. Instead continue waiting until + // timeout. See golang.org/issue/13281. + continue + } + return resp, nil } - return msg, nil } -func (c *UDPConn) writeDNSQuery(msg *dnsMsg) error { - b, ok := msg.Pack() +func (c *TCPConn) dnsRoundTrip(out *dnsMsg) (*dnsMsg, error) { + return dnsRoundTripTCP(c, out) +} + +// dnsRoundTripTCP implements the dnsRoundTrip interface for RFC 1035's +// "TCP usage" transport mechanism. c should be a stream-oriented connection, +// such as a *TCPConn. +func dnsRoundTripTCP(c io.ReadWriter, query *dnsMsg) (*dnsMsg, error) { + b, ok := query.Pack() if !ok { - return errors.New("cannot marshal DNS message") + return nil, errors.New("cannot marshal DNS message") } + l := len(b) + b = append([]byte{byte(l >> 8), byte(l)}, b...) if _, err := c.Write(b); err != nil { - return err + return nil, err } - return nil -} -func (c *TCPConn) readDNSResponse() (*dnsMsg, error) { - b := make([]byte, 1280) // 1280 is a reasonable initial size for IP over Ethernet, see RFC 4035 + b = make([]byte, 1280) // 1280 is a reasonable initial size for IP over Ethernet, see RFC 4035 if _, err := io.ReadFull(c, b[:2]); err != nil { return nil, err } - l := int(b[0])<<8 | int(b[1]) + l = int(b[0])<<8 | int(b[1]) if l > len(b) { b = make([]byte, l) } @@ -84,27 +106,17 @@ func (c *TCPConn) readDNSResponse() (*dnsMsg, error) { if err != nil { return nil, err } - msg := &dnsMsg{} - if !msg.Unpack(b[:n]) { + resp := &dnsMsg{} + if !resp.Unpack(b[:n]) { return nil, errors.New("cannot unmarshal DNS message") } - return msg, nil -} - -func (c *TCPConn) writeDNSQuery(msg *dnsMsg) error { - b, ok := msg.Pack() - if !ok { - return errors.New("cannot marshal DNS message") + if !resp.IsResponseTo(query) { + return nil, errors.New("invalid DNS response") } - l := uint16(len(b)) - b = append([]byte{byte(l >> 8), byte(l)}, b...) - if _, err := c.Write(b); err != nil { - return err - } - return nil + return resp, nil } -func (d *Dialer) dialDNS(network, server string) (dnsConn, error) { +func (d *Dialer) dialDNS(ctx context.Context, network, server string) (dnsConn, error) { switch network { case "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6": default: @@ -115,9 +127,9 @@ func (d *Dialer) dialDNS(network, server string) (dnsConn, error) { // call back here to translate it. The DNS config parser has // already checked that all the cfg.servers[i] are IP // addresses, which Dial will use without a DNS lookup. - c, err := d.Dial(network, server) + c, err := d.DialContext(ctx, network, server) if err != nil { - return nil, err + return nil, mapErr(err) } switch network { case "tcp", "tcp4", "tcp6": @@ -129,8 +141,8 @@ func (d *Dialer) dialDNS(network, server string) (dnsConn, error) { } // exchange sends a query on the connection and hopes for a response. -func exchange(server, name string, qtype uint16, timeout time.Duration) (*dnsMsg, error) { - d := testHookDNSDialer(timeout) +func exchange(ctx context.Context, server, name string, qtype uint16) (*dnsMsg, error) { + d := testHookDNSDialer() out := dnsMsg{ dnsMsgHdr: dnsMsgHdr{ recursion_desired: true, @@ -140,24 +152,18 @@ func exchange(server, name string, qtype uint16, timeout time.Duration) (*dnsMsg }, } for _, network := range []string{"udp", "tcp"} { - c, err := d.dialDNS(network, server) + c, err := d.dialDNS(ctx, network, server) if err != nil { return nil, err } defer c.Close() - if timeout > 0 { - c.SetDeadline(time.Now().Add(timeout)) + if d, ok := ctx.Deadline(); ok && !d.IsZero() { + c.SetDeadline(d) } out.id = uint16(rand.Int()) ^ uint16(time.Now().UnixNano()) - if err := c.writeDNSQuery(&out); err != nil { - return nil, err - } - in, err := c.readDNSResponse() + in, err := c.dnsRoundTrip(&out) if err != nil { - return nil, err - } - if in.id != out.id { - return nil, errors.New("DNS message ID mismatch") + return nil, mapErr(err) } if in.truncated { // see RFC 5966 continue @@ -169,16 +175,22 @@ func exchange(server, name string, qtype uint16, timeout time.Duration) (*dnsMsg // Do a lookup for a single name, which must be rooted // (otherwise answer will not find the answers). -func tryOneName(cfg *dnsConfig, name string, qtype uint16) (string, []dnsRR, error) { +func tryOneName(ctx context.Context, cfg *dnsConfig, name string, qtype uint16) (string, []dnsRR, error) { if len(cfg.servers) == 0 { return "", nil, &DNSError{Err: "no DNS servers", Name: name} } - timeout := time.Duration(cfg.timeout) * time.Second + + deadline := time.Now().Add(cfg.timeout) + if old, ok := ctx.Deadline(); !ok || deadline.Before(old) { + var cancel context.CancelFunc + ctx, cancel = context.WithDeadline(ctx, deadline) + defer cancel() + } + var lastErr error for i := 0; i < cfg.attempts; i++ { for _, server := range cfg.servers { - server = JoinHostPort(server, "53") - msg, err := exchange(server, name, qtype, timeout) + msg, err := exchange(ctx, server, name, qtype) if err != nil { lastErr = &DNSError{ Err: err.Error(), @@ -190,6 +202,12 @@ func tryOneName(cfg *dnsConfig, name string, qtype uint16) (string, []dnsRR, err } continue } + // libresolv continues to the next server when it receives + // an invalid referral response. See golang.org/issue/15434. + if msg.rcode == dnsRcodeSuccess && !msg.authoritative && !msg.recursion_available && len(msg.answer) == 0 && len(msg.extra) == 0 { + lastErr = &DNSError{Err: "lame referral", Name: name, Server: server} + continue + } cname, rrs, err := answer(name, server, msg, qtype) // If answer errored for rcodes dnsRcodeSuccess or dnsRcodeNameError, // it means the response in msg was not useful and trying another @@ -229,7 +247,6 @@ type resolverConfig struct { // time to recheck resolv.conf. ch chan struct{} // guards lastChecked and modTime lastChecked time.Time // last time resolv.conf was checked - modTime time.Time // time of resolv.conf modification mu sync.RWMutex // protects dnsConfig dnsConfig *dnsConfig // parsed resolv.conf structure used in lookups @@ -239,16 +256,12 @@ var resolvConf resolverConfig // init initializes conf and is only called via conf.initOnce. func (conf *resolverConfig) init() { - // Set dnsConfig, modTime, and lastChecked so we don't parse + // Set dnsConfig and lastChecked so we don't parse // resolv.conf twice the first time. conf.dnsConfig = systemConf().resolv if conf.dnsConfig == nil { conf.dnsConfig = dnsReadConfig("/etc/resolv.conf") } - - if fi, err := os.Stat("/etc/resolv.conf"); err == nil { - conf.modTime = fi.ModTime() - } conf.lastChecked = time.Now() // Prepare ch so that only one update of resolverConfig may @@ -274,17 +287,12 @@ func (conf *resolverConfig) tryUpdate(name string) { } conf.lastChecked = now + var mtime time.Time if fi, err := os.Stat(name); err == nil { - if fi.ModTime().Equal(conf.modTime) { - return - } - conf.modTime = fi.ModTime() - } else { - // If modTime wasn't set prior, assume nothing has changed. - if conf.modTime.IsZero() { - return - } - conf.modTime = time.Time{} + mtime = fi.ModTime() + } + if mtime.Equal(conf.dnsConfig.mtime) { + return } dnsConf := dnsReadConfig(name) @@ -306,7 +314,7 @@ func (conf *resolverConfig) releaseSema() { <-conf.ch } -func lookup(name string, qtype uint16) (cname string, rrs []dnsRR, err error) { +func lookup(ctx context.Context, name string, qtype uint16) (cname string, rrs []dnsRR, err error) { if !isDomainName(name) { return "", nil, &DNSError{Err: "invalid domain name", Name: name} } @@ -315,7 +323,7 @@ func lookup(name string, qtype uint16) (cname string, rrs []dnsRR, err error) { conf := resolvConf.dnsConfig resolvConf.mu.RUnlock() for _, fqdn := range conf.nameList(name) { - cname, rrs, err = tryOneName(conf, fqdn, qtype) + cname, rrs, err = tryOneName(ctx, conf, fqdn, qtype) if err == nil { break } @@ -329,30 +337,47 @@ func lookup(name string, qtype uint16) (cname string, rrs []dnsRR, err error) { return } +// avoidDNS reports whether this is a hostname for which we should not +// use DNS. Currently this includes only .onion and .local names, +// per RFC 7686 and RFC 6762, respectively. See golang.org/issue/13705. +func avoidDNS(name string) bool { + if name == "" { + return true + } + if name[len(name)-1] == '.' { + name = name[:len(name)-1] + } + return stringsHasSuffixFold(name, ".onion") || stringsHasSuffixFold(name, ".local") +} + // nameList returns a list of names for sequential DNS queries. func (conf *dnsConfig) nameList(name string) []string { + if avoidDNS(name) { + return nil + } + // If name is rooted (trailing dot), try only that name. rooted := len(name) > 0 && name[len(name)-1] == '.' if rooted { return []string{name} } + + hasNdots := count(name, '.') >= conf.ndots + name += "." + // Build list of search choices. names := make([]string, 0, 1+len(conf.search)) // If name has enough dots, try unsuffixed first. - if count(name, '.') >= conf.ndots { - names = append(names, name+".") + if hasNdots { + names = append(names, name) } // Try suffixes. for _, suffix := range conf.search { - suffixed := name + "." + suffix - if suffixed[len(suffixed)-1] != '.' { - suffixed += "." - } - names = append(names, suffixed) + names = append(names, name+suffix) } // Try unsuffixed, if not tried first above. - if count(name, '.') < conf.ndots { - names = append(names, name+".") + if !hasNdots { + names = append(names, name) } return names } @@ -392,11 +417,11 @@ func (o hostLookupOrder) String() string { // Normally we let cgo use the C library resolver instead of // depending on our lookup code, so that Go and C get the same // answers. -func goLookupHost(name string) (addrs []string, err error) { - return goLookupHostOrder(name, hostLookupFilesDNS) +func goLookupHost(ctx context.Context, name string) (addrs []string, err error) { + return goLookupHostOrder(ctx, name, hostLookupFilesDNS) } -func goLookupHostOrder(name string, order hostLookupOrder) (addrs []string, err error) { +func goLookupHostOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []string, err error) { if order == hostLookupFilesDNS || order == hostLookupFiles { // Use entries from /etc/hosts if they match. addrs = lookupStaticHost(name) @@ -404,7 +429,7 @@ func goLookupHostOrder(name string, order hostLookupOrder) (addrs []string, err return } } - ips, err := goLookupIPOrder(name, order) + ips, err := goLookupIPOrder(ctx, name, order) if err != nil { return } @@ -430,11 +455,11 @@ func goLookupIPFiles(name string) (addrs []IPAddr) { // goLookupIP is the native Go implementation of LookupIP. // The libc versions are in cgo_*.go. -func goLookupIP(name string) (addrs []IPAddr, err error) { - return goLookupIPOrder(name, hostLookupFilesDNS) +func goLookupIP(ctx context.Context, name string) (addrs []IPAddr, err error) { + return goLookupIPOrder(ctx, name, hostLookupFilesDNS) } -func goLookupIPOrder(name string, order hostLookupOrder) (addrs []IPAddr, err error) { +func goLookupIPOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []IPAddr, err error) { if order == hostLookupFilesDNS || order == hostLookupFiles { addrs = goLookupIPFiles(name) if len(addrs) > 0 || order == hostLookupFiles { @@ -459,7 +484,7 @@ func goLookupIPOrder(name string, order hostLookupOrder) (addrs []IPAddr, err er for _, fqdn := range conf.nameList(name) { for _, qtype := range qtypes { go func(qtype uint16) { - _, rrs, err := tryOneName(conf, fqdn, qtype) + _, rrs, err := tryOneName(ctx, conf, fqdn, qtype) lane <- racer{fqdn, rrs, err} }(qtype) } @@ -502,8 +527,8 @@ func goLookupIPOrder(name string, order hostLookupOrder) (addrs []IPAddr, err er // Normally we let cgo use the C library resolver instead of // depending on our lookup code, so that Go and C get the same // answers. -func goLookupCNAME(name string) (cname string, err error) { - _, rrs, err := lookup(name, dnsTypeCNAME) +func goLookupCNAME(ctx context.Context, name string) (cname string, err error) { + _, rrs, err := lookup(ctx, name, dnsTypeCNAME) if err != nil { return } @@ -516,7 +541,7 @@ func goLookupCNAME(name string) (cname string, err error) { // only if cgoLookupPTR is the stub in cgo_stub.go). // Normally we let cgo use the C library resolver instead of depending // on our lookup code, so that Go and C get the same answers. -func goLookupPTR(addr string) ([]string, error) { +func goLookupPTR(ctx context.Context, addr string) ([]string, error) { names := lookupStaticAddr(addr) if len(names) > 0 { return names, nil @@ -525,7 +550,7 @@ func goLookupPTR(addr string) ([]string, error) { if err != nil { return nil, err } - _, rrs, err := lookup(arpa, dnsTypePTR) + _, rrs, err := lookup(ctx, arpa, dnsTypePTR) if err != nil { return nil, err } diff --git a/libgo/go/net/dnsclient_unix_test.go b/libgo/go/net/dnsclient_unix_test.go index 934f25b..09bbd48 100644 --- a/libgo/go/net/dnsclient_unix_test.go +++ b/libgo/go/net/dnsclient_unix_test.go @@ -7,7 +7,9 @@ package net import ( + "context" "fmt" + "internal/testenv" "io/ioutil" "os" "path" @@ -18,6 +20,9 @@ import ( "time" ) +// Test address from 192.0.2.0/24 block, reserved by RFC 5737 for documentation. +const TestAddr uint32 = 0xc0000201 + var dnsTransportFallbackTests = []struct { server string name string @@ -32,13 +37,12 @@ var dnsTransportFallbackTests = []struct { } func TestDNSTransportFallback(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) for _, tt := range dnsTransportFallbackTests { - timeout := time.Duration(tt.timeout) * time.Second - msg, err := exchange(tt.server, tt.name, tt.qtype, timeout) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(tt.timeout)*time.Second) + defer cancel() + msg, err := exchange(ctx, tt.server, tt.name, tt.qtype) if err != nil { t.Error(err) continue @@ -67,20 +71,20 @@ var specialDomainNameTests = []struct { // Name resolution APIs and libraries should recognize the // followings as special and should not send any queries. - // Though, we test those names here for verifying nagative + // Though, we test those names here for verifying negative // answers at DNS query-response interaction level. {"localhost.", dnsTypeALL, dnsRcodeNameError}, {"invalid.", dnsTypeALL, dnsRcodeNameError}, } func TestSpecialDomainName(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) server := "8.8.8.8:53" for _, tt := range specialDomainNameTests { - msg, err := exchange(server, tt.name, tt.qtype, 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + msg, err := exchange(ctx, server, tt.name, tt.qtype) if err != nil { t.Error(err) continue @@ -94,6 +98,57 @@ func TestSpecialDomainName(t *testing.T) { } } +// Issue 13705: don't try to resolve onion addresses, etc +func TestAvoidDNSName(t *testing.T) { + tests := []struct { + name string + avoid bool + }{ + {"foo.com", false}, + {"foo.com.", false}, + + {"foo.onion.", true}, + {"foo.onion", true}, + {"foo.ONION", true}, + {"foo.ONION.", true}, + + {"foo.local.", true}, + {"foo.local", true}, + {"foo.LOCAL", true}, + {"foo.LOCAL.", true}, + + {"", true}, // will be rejected earlier too + + // Without stuff before onion/local, they're fine to + // use DNS. With a search path, + // "onion.vegegtables.com" can use DNS. Without a + // search path (or with a trailing dot), the queries + // are just kinda useless, but don't reveal anything + // private. + {"local", false}, + {"onion", false}, + {"local.", false}, + {"onion.", false}, + } + for _, tt := range tests { + got := avoidDNS(tt.name) + if got != tt.avoid { + t.Errorf("avoidDNS(%q) = %v; want %v", tt.name, got, tt.avoid) + } + } +} + +// Issue 13705: don't try to resolve onion addresses, etc +func TestLookupTorOnion(t *testing.T) { + addrs, err := goLookupIP(context.Background(), "foo.onion") + if len(addrs) > 0 { + t.Errorf("unexpected addresses: %v", addrs) + } + if err != nil { + t.Fatalf("lookup = %v; want nil", err) + } +} + type resolvConfTest struct { dir string path string @@ -124,20 +179,20 @@ func (conf *resolvConfTest) writeAndUpdate(lines []string) error { return err } f.Close() - if err := conf.forceUpdate(conf.path); err != nil { + if err := conf.forceUpdate(conf.path, time.Now().Add(time.Hour)); err != nil { return err } return nil } -func (conf *resolvConfTest) forceUpdate(name string) error { +func (conf *resolvConfTest) forceUpdate(name string, lastChecked time.Time) error { dnsConf := dnsReadConfig(name) conf.mu.Lock() conf.dnsConfig = dnsConf conf.mu.Unlock() for i := 0; i < 5; i++ { if conf.tryAcquireSema() { - conf.lastChecked = time.Time{} + conf.lastChecked = lastChecked conf.releaseSema() return nil } @@ -153,7 +208,7 @@ func (conf *resolvConfTest) servers() []string { } func (conf *resolvConfTest) teardown() error { - err := conf.forceUpdate("/etc/resolv.conf") + err := conf.forceUpdate("/etc/resolv.conf", time.Time{}) os.RemoveAll(conf.dir) return err } @@ -166,7 +221,7 @@ var updateResolvConfTests = []struct { { name: "golang.org", lines: []string{"nameserver 8.8.8.8"}, - servers: []string{"8.8.8.8"}, + servers: []string{"8.8.8.8:53"}, }, { name: "", @@ -176,14 +231,12 @@ var updateResolvConfTests = []struct { { name: "www.example.com", lines: []string{"nameserver 8.8.4.4"}, - servers: []string{"8.8.4.4"}, + servers: []string{"8.8.4.4:53"}, }, } func TestUpdateResolvConf(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) conf, err := newResolvConfTest() if err != nil { @@ -203,7 +256,7 @@ func TestUpdateResolvConf(t *testing.T) { for j := 0; j < N; j++ { go func(name string) { defer wg.Done() - ips, err := goLookupIP(name) + ips, err := goLookupIP(context.Background(), name) if err != nil { t.Error(err) return @@ -338,9 +391,7 @@ var goLookupIPWithResolverConfigTests = []struct { } func TestGoLookupIPWithResolverConfig(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) conf, err := newResolvConfTest() if err != nil { @@ -353,10 +404,15 @@ func TestGoLookupIPWithResolverConfig(t *testing.T) { t.Error(err) continue } - conf.tryUpdate(conf.path) - addrs, err := goLookupIP(tt.name) + addrs, err := goLookupIP(context.Background(), tt.name) if err != nil { - if err, ok := err.(*DNSError); !ok || (err.Name != tt.error.(*DNSError).Name || err.Server != tt.error.(*DNSError).Server || err.IsTimeout != tt.error.(*DNSError).IsTimeout) { + // This test uses external network connectivity. + // We need to take care with errors on both + // DNS message exchange layer and DNS + // transport layer because goLookupIP may fail + // when the IP connectivty on node under test + // gets lost during its run. + if err, ok := err.(*DNSError); !ok || tt.error != nil && (err.Name != tt.error.(*DNSError).Name || err.Server != tt.error.(*DNSError).Server || err.IsTimeout != tt.error.(*DNSError).IsTimeout) { t.Errorf("got %v; want %v", err, tt.error) } continue @@ -380,9 +436,7 @@ func TestGoLookupIPWithResolverConfig(t *testing.T) { // Test that goLookupIPOrder falls back to the host file when no DNS servers are available. func TestGoLookupIPOrderFallbackToFile(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) // Add a config that simulates no dns servers being available. conf, err := newResolvConfTest() @@ -392,7 +446,6 @@ func TestGoLookupIPOrderFallbackToFile(t *testing.T) { if err := conf.writeAndUpdate([]string{}); err != nil { t.Fatal(err) } - conf.tryUpdate(conf.path) // Redirect host file lookups. defer func(orig string) { testHookHostsPath = orig }(testHookHostsPath) testHookHostsPath = "testdata/hosts" @@ -400,15 +453,15 @@ func TestGoLookupIPOrderFallbackToFile(t *testing.T) { for _, order := range []hostLookupOrder{hostLookupFilesDNS, hostLookupDNSFiles} { name := fmt.Sprintf("order %v", order) - // First ensure that we get an error when contacting a non-existant host. - _, err := goLookupIPOrder("notarealhost", order) + // First ensure that we get an error when contacting a non-existent host. + _, err := goLookupIPOrder(context.Background(), "notarealhost", order) if err == nil { t.Errorf("%s: expected error while looking up name not in hosts file", name) continue } // Now check that we get an address when the name appears in the hosts file. - addrs, err := goLookupIPOrder("thor", order) // entry is in "testdata/hosts" + addrs, err := goLookupIPOrder(context.Background(), "thor", order) // entry is in "testdata/hosts" if err != nil { t.Errorf("%s: expected to successfully lookup host entry", name) continue @@ -444,10 +497,10 @@ func TestErrorForOriginalNameWhenSearching(t *testing.T) { t.Fatal(err) } - d := &fakeDNSConn{} - testHookDNSDialer = func(time.Duration) dnsDialer { return d } + d := &fakeDNSDialer{} + testHookDNSDialer = func() dnsDialer { return d } - d.rh = func(q *dnsMsg) (*dnsMsg, error) { + d.rh = func(s string, q *dnsMsg) (*dnsMsg, error) { r := &dnsMsg{ dnsMsgHdr: dnsMsgHdr{ id: q.id, @@ -464,7 +517,7 @@ func TestErrorForOriginalNameWhenSearching(t *testing.T) { return r, nil } - _, err = goLookupIP(fqdn) + _, err = goLookupIP(context.Background(), fqdn) if err == nil { t.Fatal("expected an error") } @@ -475,19 +528,83 @@ func TestErrorForOriginalNameWhenSearching(t *testing.T) { } } +// Issue 15434. If a name server gives a lame referral, continue to the next. +func TestIgnoreLameReferrals(t *testing.T) { + origTestHookDNSDialer := testHookDNSDialer + defer func() { testHookDNSDialer = origTestHookDNSDialer }() + + conf, err := newResolvConfTest() + if err != nil { + t.Fatal(err) + } + defer conf.teardown() + + if err := conf.writeAndUpdate([]string{"nameserver 192.0.2.1", "nameserver 192.0.2.2"}); err != nil { + t.Fatal(err) + } + + d := &fakeDNSDialer{} + testHookDNSDialer = func() dnsDialer { return d } + + d.rh = func(s string, q *dnsMsg) (*dnsMsg, error) { + t.Log(s, q) + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + }, + question: q.question, + } + + if s == "192.0.2.2:53" { + r.recursion_available = true + if q.question[0].Qtype == dnsTypeA { + r.answer = []dnsRR{ + &dnsRR_A{ + Hdr: dnsRR_Header{ + Name: q.question[0].Name, + Rrtype: dnsTypeA, + Class: dnsClassINET, + Rdlength: 4, + }, + A: TestAddr, + }, + } + } + } + + return r, nil + } + + addrs, err := goLookupIP(context.Background(), "www.golang.org") + if err != nil { + t.Fatal(err) + } + + if got := len(addrs); got != 1 { + t.Fatalf("got %d addresses, want 1", got) + } + + if got, want := addrs[0].String(), "192.0.2.1"; got != want { + t.Fatalf("got address %v, want %v", got, want) + } +} + func BenchmarkGoLookupIP(b *testing.B) { testHookUninstaller.Do(uninstallTestHooks) + ctx := context.Background() for i := 0; i < b.N; i++ { - goLookupIP("www.example.com") + goLookupIP(ctx, "www.example.com") } } func BenchmarkGoLookupIPNoSuchHost(b *testing.B) { testHookUninstaller.Do(uninstallTestHooks) + ctx := context.Background() for i := 0; i < b.N; i++ { - goLookupIP("some.nonexistent") + goLookupIP(ctx, "some.nonexistent") } } @@ -507,22 +624,25 @@ func BenchmarkGoLookupIPWithBrokenNameServer(b *testing.B) { if err := conf.writeAndUpdate(lines); err != nil { b.Fatal(err) } + ctx := context.Background() for i := 0; i < b.N; i++ { - goLookupIP("www.example.com") + goLookupIP(ctx, "www.example.com") } } -type fakeDNSConn struct { - // last query - qmu sync.Mutex // guards q - q *dnsMsg +type fakeDNSDialer struct { // reply handler - rh func(*dnsMsg) (*dnsMsg, error) + rh func(s string, q *dnsMsg) (*dnsMsg, error) } -func (f *fakeDNSConn) dialDNS(n, s string) (dnsConn, error) { - return f, nil +func (f *fakeDNSDialer) dialDNS(_ context.Context, n, s string) (dnsConn, error) { + return &fakeDNSConn{f.rh, s}, nil +} + +type fakeDNSConn struct { + rh func(s string, q *dnsMsg) (*dnsMsg, error) + s string } func (f *fakeDNSConn) Close() error { @@ -533,16 +653,74 @@ func (f *fakeDNSConn) SetDeadline(time.Time) error { return nil } -func (f *fakeDNSConn) writeDNSQuery(q *dnsMsg) error { - f.qmu.Lock() - defer f.qmu.Unlock() - f.q = q - return nil +func (f *fakeDNSConn) dnsRoundTrip(q *dnsMsg) (*dnsMsg, error) { + return f.rh(f.s, q) } -func (f *fakeDNSConn) readDNSResponse() (*dnsMsg, error) { - f.qmu.Lock() - q := f.q - f.qmu.Unlock() - return f.rh(q) +// UDP round-tripper algorithm should ignore invalid DNS responses (issue 13281). +func TestIgnoreDNSForgeries(t *testing.T) { + c, s := Pipe() + go func() { + b := make([]byte, 512) + n, err := s.Read(b) + if err != nil { + t.Fatal(err) + } + + msg := &dnsMsg{} + if !msg.Unpack(b[:n]) { + t.Fatal("invalid DNS query") + } + + s.Write([]byte("garbage DNS response packet")) + + msg.response = true + msg.id++ // make invalid ID + b, ok := msg.Pack() + if !ok { + t.Fatal("failed to pack DNS response") + } + s.Write(b) + + msg.id-- // restore original ID + msg.answer = []dnsRR{ + &dnsRR_A{ + Hdr: dnsRR_Header{ + Name: "www.example.com.", + Rrtype: dnsTypeA, + Class: dnsClassINET, + Rdlength: 4, + }, + A: TestAddr, + }, + } + + b, ok = msg.Pack() + if !ok { + t.Fatal("failed to pack DNS response") + } + s.Write(b) + }() + + msg := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: 42, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + }, + } + + resp, err := dnsRoundTripUDP(c, msg) + if err != nil { + t.Fatalf("dnsRoundTripUDP failed: %v", err) + } + + if got := resp.answer[0].(*dnsRR_A).A; got != TestAddr { + t.Errorf("got address %v, want %v", got, TestAddr) + } } diff --git a/libgo/go/net/dnsconfig_unix.go b/libgo/go/net/dnsconfig_unix.go index 6073fdb..aec575e 100644 --- a/libgo/go/net/dnsconfig_unix.go +++ b/libgo/go/net/dnsconfig_unix.go @@ -8,36 +8,52 @@ package net -var defaultNS = []string{"127.0.0.1", "::1"} +import ( + "os" + "time" +) + +var ( + defaultNS = []string{"127.0.0.1:53", "[::1]:53"} + getHostname = os.Hostname // variable for testing +) type dnsConfig struct { - servers []string // servers to use - search []string // suffixes to append to local name - ndots int // number of dots in name to trigger absolute lookup - timeout int // seconds before giving up on packet - attempts int // lost packets before giving up on server - rotate bool // round robin among servers - unknownOpt bool // anything unknown was encountered - lookup []string // OpenBSD top-level database "lookup" order - err error // any error that occurs during open of resolv.conf + servers []string // server addresses (in host:port form) to use + search []string // rooted suffixes to append to local name + ndots int // number of dots in name to trigger absolute lookup + timeout time.Duration // wait before giving up on a query, including retries + attempts int // lost packets before giving up on server + rotate bool // round robin among servers + unknownOpt bool // anything unknown was encountered + lookup []string // OpenBSD top-level database "lookup" order + err error // any error that occurs during open of resolv.conf + mtime time.Time // time of resolv.conf modification } // See resolv.conf(5) on a Linux machine. -// TODO(rsc): Supposed to call uname() and chop the beginning -// of the host name to get the default search domain. func dnsReadConfig(filename string) *dnsConfig { conf := &dnsConfig{ ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, } file, err := open(filename) if err != nil { conf.servers = defaultNS + conf.search = dnsDefaultSearch() conf.err = err return conf } defer file.close() + if fi, err := file.file.Stat(); err == nil { + conf.mtime = fi.ModTime() + } else { + conf.servers = defaultNS + conf.search = dnsDefaultSearch() + conf.err = err + return conf + } for line, ok := file.readLine(); ok; line, ok = file.readLine() { if len(line) > 0 && (line[0] == ';' || line[0] == '#') { // comment. @@ -51,24 +67,24 @@ func dnsReadConfig(filename string) *dnsConfig { case "nameserver": // add one name server if len(f) > 1 && len(conf.servers) < 3 { // small, but the standard limit // One more check: make sure server name is - // just an IP address. Otherwise we need DNS + // just an IP address. Otherwise we need DNS // to look it up. if parseIPv4(f[1]) != nil { - conf.servers = append(conf.servers, f[1]) + conf.servers = append(conf.servers, JoinHostPort(f[1], "53")) } else if ip, _ := parseIPv6(f[1], true); ip != nil { - conf.servers = append(conf.servers, f[1]) + conf.servers = append(conf.servers, JoinHostPort(f[1], "53")) } } case "domain": // set search path to just this domain if len(f) > 1 { - conf.search = []string{f[1]} + conf.search = []string{ensureRooted(f[1])} } case "search": // set search path to given servers conf.search = make([]string, len(f)-1) for i := 0; i < len(conf.search); i++ { - conf.search[i] = f[i+1] + conf.search[i] = ensureRooted(f[i+1]) } case "options": // magic options @@ -85,7 +101,7 @@ func dnsReadConfig(filename string) *dnsConfig { if n < 1 { n = 1 } - conf.timeout = n + conf.timeout = time.Duration(n) * time.Second case hasPrefix(s, "attempts:"): n, _, _ := dtoi(s, 9) if n < 1 { @@ -112,9 +128,31 @@ func dnsReadConfig(filename string) *dnsConfig { if len(conf.servers) == 0 { conf.servers = defaultNS } + if len(conf.search) == 0 { + conf.search = dnsDefaultSearch() + } return conf } +func dnsDefaultSearch() []string { + hn, err := getHostname() + if err != nil { + // best effort + return nil + } + if i := byteIndex(hn, '.'); i >= 0 && i < len(hn)-1 { + return []string{ensureRooted(hn[i+1:])} + } + return nil +} + func hasPrefix(s, prefix string) bool { return len(s) >= len(prefix) && s[:len(prefix)] == prefix } + +func ensureRooted(s string) string { + if len(s) > 0 && s[len(s)-1] == '.' { + return s + } + return s + "." +} diff --git a/libgo/go/net/dnsconfig_unix_test.go b/libgo/go/net/dnsconfig_unix_test.go index c8eed61..9fd6dbf 100644 --- a/libgo/go/net/dnsconfig_unix_test.go +++ b/libgo/go/net/dnsconfig_unix_test.go @@ -7,9 +7,11 @@ package net import ( + "errors" "os" "reflect" "testing" + "time" ) var dnsReadConfigTests = []struct { @@ -19,10 +21,10 @@ var dnsReadConfigTests = []struct { { name: "testdata/resolv.conf", want: &dnsConfig{ - servers: []string{"8.8.8.8", "2001:4860:4860::8888", "fe80::1%lo0"}, - search: []string{"localdomain"}, + servers: []string{"8.8.8.8:53", "[2001:4860:4860::8888]:53", "[fe80::1%lo0]:53"}, + search: []string{"localdomain."}, ndots: 5, - timeout: 10, + timeout: 10 * time.Second, attempts: 3, rotate: true, unknownOpt: true, // the "options attempts 3" line @@ -31,20 +33,20 @@ var dnsReadConfigTests = []struct { { name: "testdata/domain-resolv.conf", want: &dnsConfig{ - servers: []string{"8.8.8.8"}, - search: []string{"localdomain"}, + servers: []string{"8.8.8.8:53"}, + search: []string{"localdomain."}, ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, }, }, { name: "testdata/search-resolv.conf", want: &dnsConfig{ - servers: []string{"8.8.8.8"}, - search: []string{"test", "invalid"}, + servers: []string{"8.8.8.8:53"}, + search: []string{"test.", "invalid."}, ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, }, }, @@ -53,29 +55,35 @@ var dnsReadConfigTests = []struct { want: &dnsConfig{ servers: defaultNS, ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, + search: []string{"domain.local."}, }, }, { name: "testdata/openbsd-resolv.conf", want: &dnsConfig{ ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, lookup: []string{"file", "bind"}, - servers: []string{"169.254.169.254", "10.240.0.1"}, + servers: []string{"169.254.169.254:53", "10.240.0.1:53"}, search: []string{"c.symbolic-datum-552.internal."}, }, }, } func TestDNSReadConfig(t *testing.T) { + origGetHostname := getHostname + defer func() { getHostname = origGetHostname }() + getHostname = func() (string, error) { return "host.domain.local", nil } + for _, tt := range dnsReadConfigTests { conf := dnsReadConfig(tt.name) if conf.err != nil { t.Fatal(conf.err) } + conf.mtime = time.Time{} if !reflect.DeepEqual(conf, tt.want) { t.Errorf("%s:\ngot: %+v\nwant: %+v", tt.name, conf, tt.want) } @@ -83,6 +91,10 @@ func TestDNSReadConfig(t *testing.T) { } func TestDNSReadMissingFile(t *testing.T) { + origGetHostname := getHostname + defer func() { getHostname = origGetHostname }() + getHostname = func() (string, error) { return "host.domain.local", nil } + conf := dnsReadConfig("a-nonexistent-file") if !os.IsNotExist(conf.err) { t.Errorf("missing resolv.conf:\ngot: %v\nwant: %v", conf.err, os.ErrNotExist) @@ -91,10 +103,54 @@ func TestDNSReadMissingFile(t *testing.T) { want := &dnsConfig{ servers: defaultNS, ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, + search: []string{"domain.local."}, } if !reflect.DeepEqual(conf, want) { t.Errorf("missing resolv.conf:\ngot: %+v\nwant: %+v", conf, want) } } + +var dnsDefaultSearchTests = []struct { + name string + err error + want []string +}{ + { + name: "host.long.domain.local", + want: []string{"long.domain.local."}, + }, + { + name: "host.local", + want: []string{"local."}, + }, + { + name: "host", + want: nil, + }, + { + name: "host.domain.local", + err: errors.New("errored"), + want: nil, + }, + { + // ensures we don't return []string{""} + // which causes duplicate lookups + name: "foo.", + want: nil, + }, +} + +func TestDNSDefaultSearch(t *testing.T) { + origGetHostname := getHostname + defer func() { getHostname = origGetHostname }() + + for _, tt := range dnsDefaultSearchTests { + getHostname = func() (string, error) { return tt.name, tt.err } + got := dnsDefaultSearch() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("dnsDefaultSearch with hostname %q and error %+v = %q, wanted %q", tt.name, tt.err, got, tt.want) + } + } +} diff --git a/libgo/go/net/dnsmsg.go b/libgo/go/net/dnsmsg.go index 93078fe..afdb44c 100644 --- a/libgo/go/net/dnsmsg.go +++ b/libgo/go/net/dnsmsg.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// DNS packet assembly. See RFC 1035. +// DNS packet assembly. See RFC 1035. // // This is intended to support name resolution during Dial. // It doesn't have to be blazing fast. @@ -18,7 +18,7 @@ // generic pack/unpack routines. // // TODO(rsc): There are enough names defined in this file that they're all -// prefixed with dns. Perhaps put this in its own package later. +// prefixed with dns. Perhaps put this in its own package later. package net @@ -109,7 +109,7 @@ const ( // DNS queries. type dnsQuestion struct { - Name string `net:"domain-name"` // `net:"domain-name"` specifies encoding; see packers below + Name string Qtype uint16 Qclass uint16 } @@ -124,7 +124,7 @@ func (q *dnsQuestion) Walk(f func(v interface{}, name, tag string) bool) bool { // There are many types of messages, // but they all share the same header. type dnsRR_Header struct { - Name string `net:"domain-name"` + Name string Rrtype uint16 Class uint16 Ttl uint32 @@ -152,7 +152,7 @@ type dnsRR interface { type dnsRR_CNAME struct { Hdr dnsRR_Header - Cname string `net:"domain-name"` + Cname string } func (rr *dnsRR_CNAME) Header() *dnsRR_Header { @@ -163,77 +163,10 @@ func (rr *dnsRR_CNAME) Walk(f func(v interface{}, name, tag string) bool) bool { return rr.Hdr.Walk(f) && f(&rr.Cname, "Cname", "domain") } -type dnsRR_HINFO struct { - Hdr dnsRR_Header - Cpu string - Os string -} - -func (rr *dnsRR_HINFO) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_HINFO) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Cpu, "Cpu", "") && f(&rr.Os, "Os", "") -} - -type dnsRR_MB struct { - Hdr dnsRR_Header - Mb string `net:"domain-name"` -} - -func (rr *dnsRR_MB) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_MB) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Mb, "Mb", "domain") -} - -type dnsRR_MG struct { - Hdr dnsRR_Header - Mg string `net:"domain-name"` -} - -func (rr *dnsRR_MG) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_MG) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Mg, "Mg", "domain") -} - -type dnsRR_MINFO struct { - Hdr dnsRR_Header - Rmail string `net:"domain-name"` - Email string `net:"domain-name"` -} - -func (rr *dnsRR_MINFO) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_MINFO) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Rmail, "Rmail", "domain") && f(&rr.Email, "Email", "domain") -} - -type dnsRR_MR struct { - Hdr dnsRR_Header - Mr string `net:"domain-name"` -} - -func (rr *dnsRR_MR) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_MR) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Mr, "Mr", "domain") -} - type dnsRR_MX struct { Hdr dnsRR_Header Pref uint16 - Mx string `net:"domain-name"` + Mx string } func (rr *dnsRR_MX) Header() *dnsRR_Header { @@ -246,7 +179,7 @@ func (rr *dnsRR_MX) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_NS struct { Hdr dnsRR_Header - Ns string `net:"domain-name"` + Ns string } func (rr *dnsRR_NS) Header() *dnsRR_Header { @@ -259,7 +192,7 @@ func (rr *dnsRR_NS) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_PTR struct { Hdr dnsRR_Header - Ptr string `net:"domain-name"` + Ptr string } func (rr *dnsRR_PTR) Header() *dnsRR_Header { @@ -272,8 +205,8 @@ func (rr *dnsRR_PTR) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_SOA struct { Hdr dnsRR_Header - Ns string `net:"domain-name"` - Mbox string `net:"domain-name"` + Ns string + Mbox string Serial uint32 Refresh uint32 Retry uint32 @@ -315,7 +248,7 @@ func (rr *dnsRR_TXT) Walk(f func(v interface{}, name, tag string) bool) bool { if !f(&txt, "Txt", "") { return false } - // more bytes than rr.Hdr.Rdlength said there woudld be + // more bytes than rr.Hdr.Rdlength said there would be if rr.Hdr.Rdlength-n < uint16(len(txt))+1 { return false } @@ -330,7 +263,7 @@ type dnsRR_SRV struct { Priority uint16 Weight uint16 Port uint16 - Target string `net:"domain-name"` + Target string } func (rr *dnsRR_SRV) Header() *dnsRR_Header { @@ -347,7 +280,7 @@ func (rr *dnsRR_SRV) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_A struct { Hdr dnsRR_Header - A uint32 `net:"ipv4"` + A uint32 } func (rr *dnsRR_A) Header() *dnsRR_Header { @@ -360,7 +293,7 @@ func (rr *dnsRR_A) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_AAAA struct { Hdr dnsRR_Header - AAAA [16]byte `net:"ipv6"` + AAAA [16]byte } func (rr *dnsRR_AAAA) Header() *dnsRR_Header { @@ -376,17 +309,12 @@ func (rr *dnsRR_AAAA) Walk(f func(v interface{}, name, tag string) bool) bool { // All the packers and unpackers take a (msg []byte, off int) // and return (off1 int, ok bool). If they return ok==false, they // also return off1==len(msg), so that the next unpacker will -// also fail. This lets us avoid checks of ok until the end of a +// also fail. This lets us avoid checks of ok until the end of a // packing sequence. // Map of constructors for each RR wire type. var rr_mk = map[int]func() dnsRR{ dnsTypeCNAME: func() dnsRR { return new(dnsRR_CNAME) }, - dnsTypeHINFO: func() dnsRR { return new(dnsRR_HINFO) }, - dnsTypeMB: func() dnsRR { return new(dnsRR_MB) }, - dnsTypeMG: func() dnsRR { return new(dnsRR_MG) }, - dnsTypeMINFO: func() dnsRR { return new(dnsRR_MINFO) }, - dnsTypeMR: func() dnsRR { return new(dnsRR_MR) }, dnsTypeMX: func() dnsRR { return new(dnsRR_MX) }, dnsTypeNS: func() dnsRR { return new(dnsRR_NS) }, dnsTypePTR: func() dnsRR { return new(dnsRR_PTR) }, @@ -399,13 +327,20 @@ var rr_mk = map[int]func() dnsRR{ // Pack a domain name s into msg[off:]. // Domain names are a sequence of counted strings -// split at the dots. They end with a zero-length string. +// split at the dots. They end with a zero-length string. func packDomainName(s string, msg []byte, off int) (off1 int, ok bool) { // Add trailing dot to canonicalize name. if n := len(s); n == 0 || s[n-1] != '.' { s += "." } + // Allow root domain. + if s == "." { + msg[off] = 0 + off++ + return off, true + } + // Each dot ends a segment of the name. // We trade each dot byte for a length byte. // There is also a trailing zero. @@ -422,8 +357,13 @@ func packDomainName(s string, msg []byte, off int) (off1 int, ok bool) { if i-begin >= 1<<6 { // top two bits of length must be clear return len(msg), false } + if i-begin == 0 { + return len(msg), false + } + msg[off] = byte(i - begin) off++ + for j := begin; j < i; j++ { msg[off] = s[j] off++ @@ -440,8 +380,8 @@ func packDomainName(s string, msg []byte, off int) (off1 int, ok bool) { // In addition to the simple sequences of counted strings above, // domain names are allowed to refer to strings elsewhere in the // packet, to avoid repeating common suffixes when returning -// many entries in a single domain. The pointers are marked -// by a length byte with the top two bits set. Ignoring those +// many entries in a single domain. The pointers are marked +// by a length byte with the top two bits set. Ignoring those // two bits, that byte and the next give a 14 bit offset from msg[0] // where we should pick up the trail. // Note that if we jump elsewhere in the packet, @@ -494,6 +434,9 @@ Loop: return "", len(msg), false } } + if len(s) == 0 { + s = "." + } if ptr == 0 { off1 = off } @@ -803,20 +746,32 @@ func (dns *dnsMsg) Pack() (msg []byte, ok bool) { // Pack it in: header and then the pieces. off := 0 off, ok = packStruct(&dh, msg, off) + if !ok { + return nil, false + } for i := 0; i < len(question); i++ { off, ok = packStruct(&question[i], msg, off) + if !ok { + return nil, false + } } for i := 0; i < len(answer); i++ { off, ok = packRR(answer[i], msg, off) + if !ok { + return nil, false + } } for i := 0; i < len(ns); i++ { off, ok = packRR(ns[i], msg, off) + if !ok { + return nil, false + } } for i := 0; i < len(extra); i++ { off, ok = packRR(extra[i], msg, off) - } - if !ok { - return nil, false + if !ok { + return nil, false + } } return msg[0:off], true } @@ -848,6 +803,9 @@ func (dns *dnsMsg) Unpack(msg []byte) bool { for i := 0; i < len(dns.question); i++ { off, ok = unpackStruct(&dns.question[i], msg, off) + if !ok { + return false + } } for i := 0; i < int(dh.Ancount); i++ { rec, off, ok = unpackRR(msg, off) @@ -904,3 +862,23 @@ func (dns *dnsMsg) String() string { } return s } + +// IsResponseTo reports whether m is an acceptable response to query. +func (m *dnsMsg) IsResponseTo(query *dnsMsg) bool { + if !m.response { + return false + } + if m.id != query.id { + return false + } + if len(m.question) != len(query.question) { + return false + } + for i, q := range m.question { + q2 := query.question[i] + if !equalASCIILabel(q.Name, q2.Name) || q.Qtype != q2.Qtype || q.Qclass != q2.Qclass { + return false + } + } + return true +} diff --git a/libgo/go/net/dnsmsg_test.go b/libgo/go/net/dnsmsg_test.go index 1078d77..25bd98c 100644 --- a/libgo/go/net/dnsmsg_test.go +++ b/libgo/go/net/dnsmsg_test.go @@ -10,6 +10,103 @@ import ( "testing" ) +func TestStructPackUnpack(t *testing.T) { + want := dnsQuestion{ + Name: ".", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + } + buf := make([]byte, 50) + n, ok := packStruct(&want, buf, 0) + if !ok { + t.Fatal("packing failed") + } + buf = buf[:n] + got := dnsQuestion{} + n, ok = unpackStruct(&got, buf, 0) + if !ok { + t.Fatal("unpacking failed") + } + if n != len(buf) { + t.Errorf("unpacked different amount than packed: got n = %d, want = %d", n, len(buf)) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got = %+v, want = %+v", got, want) + } +} + +func TestDomainNamePackUnpack(t *testing.T) { + tests := []struct { + in string + want string + ok bool + }{ + {"", ".", true}, + {".", ".", true}, + {"google..com", "", false}, + {"google.com", "google.com.", true}, + {"google..com.", "", false}, + {"google.com.", "google.com.", true}, + {".google.com.", "", false}, + {"www..google.com.", "", false}, + {"www.google.com.", "www.google.com.", true}, + } + + for _, test := range tests { + buf := make([]byte, 30) + n, ok := packDomainName(test.in, buf, 0) + if ok != test.ok { + t.Errorf("packing of %s: got ok = %t, want = %t", test.in, ok, test.ok) + continue + } + if !test.ok { + continue + } + buf = buf[:n] + got, n, ok := unpackDomainName(buf, 0) + if !ok { + t.Errorf("unpacking for %s failed", test.in) + continue + } + if n != len(buf) { + t.Errorf( + "unpacked different amount than packed for %s: got n = %d, want = %d", + test.in, + n, + len(buf), + ) + } + if got != test.want { + t.Errorf("unpacking packing of %s: got = %s, want = %s", test.in, got, test.want) + } + } +} + +func TestDNSPackUnpack(t *testing.T) { + want := dnsMsg{ + question: []dnsQuestion{{ + Name: ".", + Qtype: dnsTypeAAAA, + Qclass: dnsClassINET, + }}, + answer: []dnsRR{}, + ns: []dnsRR{}, + extra: []dnsRR{}, + } + b, ok := want.Pack() + if !ok { + t.Fatal("packing failed") + } + var got dnsMsg + ok = got.Unpack(b) + if !ok { + t.Fatal("unpacking failed") + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got = %+v, want = %+v", got, want) + } +} + func TestDNSParseSRVReply(t *testing.T) { data, err := hex.DecodeString(dnsSRVReply) if err != nil { @@ -183,6 +280,124 @@ func TestDNSParseTXTCorruptTXTLengthReply(t *testing.T) { } } +func TestIsResponseTo(t *testing.T) { + // Sample DNS query. + query := dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: 42, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + }, + } + + resp := query + resp.response = true + if !resp.IsResponseTo(&query) { + t.Error("got false, want true") + } + + badResponses := []dnsMsg{ + // Different ID. + { + dnsMsgHdr: dnsMsgHdr{ + id: 43, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + }, + }, + + // Different query name. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.google.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + }, + }, + + // Different query type. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeAAAA, + Qclass: dnsClassINET, + }, + }, + }, + + // Different query class. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassCSNET, + }, + }, + }, + + // No questions. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + }, + + // Extra questions. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + { + Name: "www.golang.org.", + Qtype: dnsTypeAAAA, + Qclass: dnsClassINET, + }, + }, + }, + } + + for i := range badResponses { + if badResponses[i].IsResponseTo(&query) { + t.Error("%v: got true, want false", i) + } + } +} + // Valid DNS SRV reply const dnsSRVReply = "0901818000010005000000000c5f786d70702d736572766572045f74637006676f6f67" + "6c6503636f6d0000210001c00c002100010000012c00210014000014950c786d70702d" + diff --git a/libgo/go/net/dnsname_test.go b/libgo/go/net/dnsname_test.go index be07dc6..bc777b8 100644 --- a/libgo/go/net/dnsname_test.go +++ b/libgo/go/net/dnsname_test.go @@ -15,7 +15,7 @@ type dnsNameTest struct { } var dnsNameTests = []dnsNameTest{ - // RFC2181, section 11. + // RFC 2181, section 11. {"_xmpp-server._tcp.google.com", true}, {"foo.com", true}, {"1foo.com", true}, diff --git a/libgo/go/net/error_plan9_test.go b/libgo/go/net/error_plan9_test.go index 495ea96..d7c7f14 100644 --- a/libgo/go/net/error_plan9_test.go +++ b/libgo/go/net/error_plan9_test.go @@ -9,6 +9,8 @@ import "syscall" var ( errTimedout = syscall.ETIMEDOUT errOpNotSupported = syscall.EPLAN9 + + abortedConnRequestErrors []error ) func isPlatformError(err error) bool { diff --git a/libgo/go/net/error_posix_test.go b/libgo/go/net/error_posix_test.go index 981cc83..b411a37 100644 --- a/libgo/go/net/error_posix_test.go +++ b/libgo/go/net/error_posix_test.go @@ -12,16 +12,6 @@ import ( "testing" ) -var ( - errTimedout = syscall.ETIMEDOUT - errOpNotSupported = syscall.EOPNOTSUPP -) - -func isPlatformError(err error) bool { - _, ok := err.(syscall.Errno) - return ok -} - func TestSpuriousENOTAVAIL(t *testing.T) { for _, tt := range []struct { error diff --git a/libgo/go/net/error_test.go b/libgo/go/net/error_test.go index 1aab14c4..d6de5a3 100644 --- a/libgo/go/net/error_test.go +++ b/libgo/go/net/error_test.go @@ -5,6 +5,7 @@ package net import ( + "context" "fmt" "io" "io/ioutil" @@ -91,9 +92,12 @@ second: case *os.SyscallError: nestedErr = err.Err goto third + case *os.PathError: // for Plan 9 + nestedErr = err.Err + goto third } switch nestedErr { - case errCanceled, errClosing, errMissingAddress: + case errCanceled, errClosing, errMissingAddress, errNoSuitableAddress: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) @@ -135,7 +139,7 @@ func TestDialError(t *testing.T) { origTestHookLookupIP := testHookLookupIP defer func() { testHookLookupIP = origTestHookLookupIP }() - testHookLookupIP = func(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { + testHookLookupIP = func(ctx context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { return nil, &DNSError{Err: "dial error test", Name: "name", Server: "server", IsTimeout: true} } sw.Set(socktest.FilterConnect, func(so *socktest.Status) (socktest.AfterFilter, error) { @@ -203,6 +207,58 @@ func TestProtocolDialError(t *testing.T) { } } +func TestDialAddrError(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9": + t.Skipf("not supported on %s", runtime.GOOS) + } + if !supportsIPv4 || !supportsIPv6 { + t.Skip("both IPv4 and IPv6 are required") + } + + for _, tt := range []struct { + network string + lit string + addr *TCPAddr + }{ + {"tcp4", "::1", nil}, + {"tcp4", "", &TCPAddr{IP: IPv6loopback}}, + // We don't test the {"tcp6", "byte sequence", nil} + // case for now because there is no easy way to + // control name resolution. + {"tcp6", "", &TCPAddr{IP: IP{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}}}, + } { + var err error + var c Conn + if tt.lit != "" { + c, err = Dial(tt.network, JoinHostPort(tt.lit, "0")) + } else { + c, err = DialTCP(tt.network, nil, tt.addr) + } + if err == nil { + c.Close() + t.Errorf("%s %q/%v: should fail", tt.network, tt.lit, tt.addr) + continue + } + if perr := parseDialError(err); perr != nil { + t.Error(perr) + continue + } + aerr, ok := err.(*OpError).Err.(*AddrError) + if !ok { + t.Errorf("%s %q/%v: should be AddrError: %v", tt.network, tt.lit, tt.addr, err) + continue + } + want := tt.lit + if tt.lit == "" { + want = tt.addr.IP.String() + } + if aerr.Addr != want { + t.Fatalf("%s: got %q; want %q", tt.network, aerr.Addr, want) + } + } +} + var listenErrorTests = []struct { network, address string }{ @@ -228,7 +284,7 @@ func TestListenError(t *testing.T) { origTestHookLookupIP := testHookLookupIP defer func() { testHookLookupIP = origTestHookLookupIP }() - testHookLookupIP = func(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { + testHookLookupIP = func(_ context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { return nil, &DNSError{Err: "listen error test", Name: "name", Server: "server", IsTimeout: true} } sw.Set(socktest.FilterListen, func(so *socktest.Status) (socktest.AfterFilter, error) { @@ -288,7 +344,7 @@ func TestListenPacketError(t *testing.T) { origTestHookLookupIP := testHookLookupIP defer func() { testHookLookupIP = origTestHookLookupIP }() - testHookLookupIP = func(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { + testHookLookupIP = func(_ context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { return nil, &DNSError{Err: "listen error test", Name: "name", Server: "server", IsTimeout: true} } @@ -413,7 +469,7 @@ second: goto third } switch nestedErr { - case errCanceled, errClosing, errTimeout, ErrWriteToConnected, io.ErrUnexpectedEOF: + case errCanceled, errClosing, errMissingAddress, errTimeout, ErrWriteToConnected, io.ErrUnexpectedEOF: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) @@ -543,6 +599,9 @@ second: case *os.SyscallError: nestedErr = err.Err goto third + case *os.PathError: // for Plan 9 + nestedErr = err.Err + goto third } switch nestedErr { case errClosing, errTimeout: @@ -703,3 +762,17 @@ func TestFileError(t *testing.T) { ln.Close() } } + +func parseLookupPortError(nestedErr error) error { + if nestedErr == nil { + return nil + } + + switch nestedErr.(type) { + case *AddrError, *DNSError: + return nil + case *os.PathError: // for Plan 9 + return nil + } + return fmt.Errorf("unexpected type on 1st nested level: %T", nestedErr) +} diff --git a/libgo/go/net/error_unix_test.go b/libgo/go/net/error_unix_test.go new file mode 100644 index 0000000..9ce9e12 --- /dev/null +++ b/libgo/go/net/error_unix_test.go @@ -0,0 +1,34 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!windows + +package net + +import ( + "os" + "syscall" +) + +var ( + errTimedout = syscall.ETIMEDOUT + errOpNotSupported = syscall.EOPNOTSUPP + + abortedConnRequestErrors = []error{syscall.ECONNABORTED} // see accept in fd_unix.go +) + +func isPlatformError(err error) bool { + _, ok := err.(syscall.Errno) + return ok +} + +func samePlatformError(err, want error) bool { + if op, ok := err.(*OpError); ok { + err = op.Err + } + if sys, ok := err.(*os.SyscallError); ok { + err = sys.Err + } + return err == want +} diff --git a/libgo/go/net/error_windows_test.go b/libgo/go/net/error_windows_test.go new file mode 100644 index 0000000..834a9de --- /dev/null +++ b/libgo/go/net/error_windows_test.go @@ -0,0 +1,19 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "syscall" + +var ( + errTimedout = syscall.ETIMEDOUT + errOpNotSupported = syscall.EOPNOTSUPP + + abortedConnRequestErrors = []error{syscall.ERROR_NETNAME_DELETED, syscall.WSAECONNRESET} // see accept in fd_windows.go +) + +func isPlatformError(err error) bool { + _, ok := err.(syscall.Errno) + return ok +} diff --git a/libgo/go/net/external_test.go b/libgo/go/net/external_test.go index d5ff2be..e18b547 100644 --- a/libgo/go/net/external_test.go +++ b/libgo/go/net/external_test.go @@ -6,15 +6,15 @@ package net import ( "fmt" + "internal/testenv" "io" "strings" "testing" ) func TestResolveGoogle(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv4 || !supportsIPv6 || !*testIPv4 || !*testIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -60,9 +60,8 @@ var dialGoogleTests = []struct { } func TestDialGoogle(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv4 || !supportsIPv6 || !*testIPv4 || !*testIPv6 { t.Skip("both IPv4 and IPv6 are required") } diff --git a/libgo/go/net/fd_mutex.go b/libgo/go/net/fd_mutex.go index 6d5509d..4591fd1 100644 --- a/libgo/go/net/fd_mutex.go +++ b/libgo/go/net/fd_mutex.go @@ -6,9 +6,9 @@ package net import "sync/atomic" -// fdMutex is a specialized synchronization primitive -// that manages lifetime of an fd and serializes access -// to Read and Write methods on netFD. +// fdMutex is a specialized synchronization primitive that manages +// lifetime of an fd and serializes access to Read, Write and Close +// methods on netFD. type fdMutex struct { state uint64 rsema uint32 @@ -34,18 +34,21 @@ const ( mutexWMask = (1<<20 - 1) << 43 ) -// Read operations must do RWLock(true)/RWUnlock(true). -// Write operations must do RWLock(false)/RWUnlock(false). -// Misc operations must do Incref/Decref. Misc operations include functions like -// setsockopt and setDeadline. They need to use Incref/Decref to ensure that -// they operate on the correct fd in presence of a concurrent Close call -// (otherwise fd can be closed under their feet). -// Close operation must do IncrefAndClose/Decref. +// Read operations must do rwlock(true)/rwunlock(true). +// +// Write operations must do rwlock(false)/rwunlock(false). +// +// Misc operations must do incref/decref. +// Misc operations include functions like setsockopt and setDeadline. +// They need to use incref/decref to ensure that they operate on the +// correct fd in presence of a concurrent close call (otherwise fd can +// be closed under their feet). +// +// Close operations must do increfAndClose/decref. -// RWLock/Incref return whether fd is open. -// RWUnlock/Decref return whether fd is closed and there are no remaining references. - -func (mu *fdMutex) Incref() bool { +// incref adds a reference to mu. +// It reports whether mu is available for reading or writing. +func (mu *fdMutex) incref() bool { for { old := atomic.LoadUint64(&mu.state) if old&mutexClosed != 0 { @@ -61,7 +64,9 @@ func (mu *fdMutex) Incref() bool { } } -func (mu *fdMutex) IncrefAndClose() bool { +// increfAndClose sets the state of mu to closed. +// It reports whether there is no remaining reference. +func (mu *fdMutex) increfAndClose() bool { for { old := atomic.LoadUint64(&mu.state) if old&mutexClosed != 0 { @@ -90,7 +95,9 @@ func (mu *fdMutex) IncrefAndClose() bool { } } -func (mu *fdMutex) Decref() bool { +// decref removes a reference from mu. +// It reports whether there is no remaining reference. +func (mu *fdMutex) decref() bool { for { old := atomic.LoadUint64(&mu.state) if old&mutexRefMask == 0 { @@ -103,7 +110,9 @@ func (mu *fdMutex) Decref() bool { } } -func (mu *fdMutex) RWLock(read bool) bool { +// lock adds a reference to mu and locks mu. +// It reports whether mu is available for reading or writing. +func (mu *fdMutex) rwlock(read bool) bool { var mutexBit, mutexWait, mutexMask uint64 var mutexSema *uint32 if read { @@ -146,7 +155,9 @@ func (mu *fdMutex) RWLock(read bool) bool { } } -func (mu *fdMutex) RWUnlock(read bool) bool { +// unlock removes a reference from mu and unlocks mu. +// It reports whether there is no remaining reference. +func (mu *fdMutex) rwunlock(read bool) bool { var mutexBit, mutexWait, mutexMask uint64 var mutexSema *uint32 if read { @@ -182,3 +193,57 @@ func (mu *fdMutex) RWUnlock(read bool) bool { // Implemented in runtime package. func runtime_Semacquire(sema *uint32) func runtime_Semrelease(sema *uint32) + +// incref adds a reference to fd. +// It returns an error when fd cannot be used. +func (fd *netFD) incref() error { + if !fd.fdmu.incref() { + return errClosing + } + return nil +} + +// decref removes a reference from fd. +// It also closes fd when the state of fd is set to closed and there +// is no remaining reference. +func (fd *netFD) decref() { + if fd.fdmu.decref() { + fd.destroy() + } +} + +// readLock adds a reference to fd and locks fd for reading. +// It returns an error when fd cannot be used for reading. +func (fd *netFD) readLock() error { + if !fd.fdmu.rwlock(true) { + return errClosing + } + return nil +} + +// readUnlock removes a reference from fd and unlocks fd for reading. +// It also closes fd when the state of fd is set to closed and there +// is no remaining reference. +func (fd *netFD) readUnlock() { + if fd.fdmu.rwunlock(true) { + fd.destroy() + } +} + +// writeLock adds a reference to fd and locks fd for writing. +// It returns an error when fd cannot be used for writing. +func (fd *netFD) writeLock() error { + if !fd.fdmu.rwlock(false) { + return errClosing + } + return nil +} + +// writeUnlock removes a reference from fd and unlocks fd for writing. +// It also closes fd when the state of fd is set to closed and there +// is no remaining reference. +func (fd *netFD) writeUnlock() { + if fd.fdmu.rwunlock(false) { + fd.destroy() + } +} diff --git a/libgo/go/net/fd_mutex_test.go b/libgo/go/net/fd_mutex_test.go index c34ec59..3542c70 100644 --- a/libgo/go/net/fd_mutex_test.go +++ b/libgo/go/net/fd_mutex_test.go @@ -14,44 +14,44 @@ import ( func TestMutexLock(t *testing.T) { var mu fdMutex - if !mu.Incref() { + if !mu.incref() { t.Fatal("broken") } - if mu.Decref() { + if mu.decref() { t.Fatal("broken") } - if !mu.RWLock(true) { + if !mu.rwlock(true) { t.Fatal("broken") } - if mu.RWUnlock(true) { + if mu.rwunlock(true) { t.Fatal("broken") } - if !mu.RWLock(false) { + if !mu.rwlock(false) { t.Fatal("broken") } - if mu.RWUnlock(false) { + if mu.rwunlock(false) { t.Fatal("broken") } } func TestMutexClose(t *testing.T) { var mu fdMutex - if !mu.IncrefAndClose() { + if !mu.increfAndClose() { t.Fatal("broken") } - if mu.Incref() { + if mu.incref() { t.Fatal("broken") } - if mu.RWLock(true) { + if mu.rwlock(true) { t.Fatal("broken") } - if mu.RWLock(false) { + if mu.rwlock(false) { t.Fatal("broken") } - if mu.IncrefAndClose() { + if mu.increfAndClose() { t.Fatal("broken") } } @@ -59,10 +59,10 @@ func TestMutexClose(t *testing.T) { func TestMutexCloseUnblock(t *testing.T) { c := make(chan bool) var mu fdMutex - mu.RWLock(true) + mu.rwlock(true) for i := 0; i < 4; i++ { go func() { - if mu.RWLock(true) { + if mu.rwlock(true) { t.Error("broken") return } @@ -76,7 +76,7 @@ func TestMutexCloseUnblock(t *testing.T) { t.Fatal("broken") default: } - mu.IncrefAndClose() // Must unblock the readers. + mu.increfAndClose() // Must unblock the readers. for i := 0; i < 4; i++ { select { case <-c: @@ -84,10 +84,10 @@ func TestMutexCloseUnblock(t *testing.T) { t.Fatal("broken") } } - if mu.Decref() { + if mu.decref() { t.Fatal("broken") } - if !mu.RWUnlock(true) { + if !mu.rwunlock(true) { t.Fatal("broken") } } @@ -103,21 +103,21 @@ func TestMutexPanic(t *testing.T) { } var mu fdMutex - ensurePanics(func() { mu.Decref() }) - ensurePanics(func() { mu.RWUnlock(true) }) - ensurePanics(func() { mu.RWUnlock(false) }) + ensurePanics(func() { mu.decref() }) + ensurePanics(func() { mu.rwunlock(true) }) + ensurePanics(func() { mu.rwunlock(false) }) - ensurePanics(func() { mu.Incref(); mu.Decref(); mu.Decref() }) - ensurePanics(func() { mu.RWLock(true); mu.RWUnlock(true); mu.RWUnlock(true) }) - ensurePanics(func() { mu.RWLock(false); mu.RWUnlock(false); mu.RWUnlock(false) }) + ensurePanics(func() { mu.incref(); mu.decref(); mu.decref() }) + ensurePanics(func() { mu.rwlock(true); mu.rwunlock(true); mu.rwunlock(true) }) + ensurePanics(func() { mu.rwlock(false); mu.rwunlock(false); mu.rwunlock(false) }) // ensure that it's still not broken - mu.Incref() - mu.Decref() - mu.RWLock(true) - mu.RWUnlock(true) - mu.RWLock(false) - mu.RWUnlock(false) + mu.incref() + mu.decref() + mu.rwlock(true) + mu.rwunlock(true) + mu.rwlock(false) + mu.rwunlock(false) } func TestMutexStress(t *testing.T) { @@ -138,16 +138,16 @@ func TestMutexStress(t *testing.T) { for i := 0; i < N; i++ { switch r.Intn(3) { case 0: - if !mu.Incref() { + if !mu.incref() { t.Error("broken") return } - if mu.Decref() { + if mu.decref() { t.Error("broken") return } case 1: - if !mu.RWLock(true) { + if !mu.rwlock(true) { t.Error("broken") return } @@ -158,12 +158,12 @@ func TestMutexStress(t *testing.T) { } readState[0]++ readState[1]++ - if mu.RWUnlock(true) { + if mu.rwunlock(true) { t.Error("broken") return } case 2: - if !mu.RWLock(false) { + if !mu.rwlock(false) { t.Error("broken") return } @@ -174,7 +174,7 @@ func TestMutexStress(t *testing.T) { } writeState[0]++ writeState[1]++ - if mu.RWUnlock(false) { + if mu.rwunlock(false) { t.Error("broken") return } @@ -186,10 +186,10 @@ func TestMutexStress(t *testing.T) { for p := 0; p < P; p++ { <-done } - if !mu.IncrefAndClose() { + if !mu.increfAndClose() { t.Fatal("broken") } - if !mu.Decref() { + if !mu.decref() { t.Fatal("broken") } } diff --git a/libgo/go/net/fd_plan9.go b/libgo/go/net/fd_plan9.go index cec8860..7533232 100644 --- a/libgo/go/net/fd_plan9.go +++ b/libgo/go/net/fd_plan9.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -32,12 +32,6 @@ func sysInit() { netdir = "/net" } -func dial(net string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { - // On plan9, use the relatively inefficient - // goroutine-racing implementation. - return dialChannel(net, ra, dialer, deadline) -} - func newFD(net, name string, ctl, data *os.File, laddr, raddr Addr) (*netFD, error) { return &netFD{net: net, n: name, dir: netdir + "/" + net + "/" + name, ctl: ctl, data: data, laddr: laddr, raddr: raddr}, nil } @@ -74,55 +68,6 @@ func (fd *netFD) destroy() { fd.data = nil } -// Add a reference to this fd. -// Returns an error if the fd cannot be used. -func (fd *netFD) incref() error { - if !fd.fdmu.Incref() { - return errClosing - } - return nil -} - -// Remove a reference to this FD and close if we've been asked to do so -// (and there are no references left). -func (fd *netFD) decref() { - if fd.fdmu.Decref() { - fd.destroy() - } -} - -// Add a reference to this fd and lock for reading. -// Returns an error if the fd cannot be used. -func (fd *netFD) readLock() error { - if !fd.fdmu.RWLock(true) { - return errClosing - } - return nil -} - -// Unlock for reading and remove a reference to this FD. -func (fd *netFD) readUnlock() { - if fd.fdmu.RWUnlock(true) { - fd.destroy() - } -} - -// Add a reference to this fd and lock for writing. -// Returns an error if the fd cannot be used. -func (fd *netFD) writeLock() error { - if !fd.fdmu.RWLock(false) { - return errClosing - } - return nil -} - -// Unlock for writing and remove a reference to this FD. -func (fd *netFD) writeUnlock() { - if fd.fdmu.RWUnlock(false) { - fd.destroy() - } -} - func (fd *netFD) Read(b []byte) (n int, err error) { if !fd.ok() || fd.data == nil { return 0, syscall.EINVAL @@ -131,7 +76,13 @@ func (fd *netFD) Read(b []byte) (n int, err error) { return 0, err } defer fd.readUnlock() + if len(b) == 0 { + return 0, nil + } n, err = fd.data.Read(b) + if isHangup(err) { + err = io.EOF + } if fd.net == "udp" && err == io.EOF { n = 0 err = nil @@ -165,7 +116,7 @@ func (fd *netFD) closeWrite() error { } func (fd *netFD) Close() error { - if !fd.fdmu.IncrefAndClose() { + if !fd.fdmu.increfAndClose() { return errClosing } if !fd.ok() { @@ -206,9 +157,7 @@ func (l *TCPListener) dup() (*os.File, error) { } func (fd *netFD) file(f *os.File, s string) (*os.File, error) { - syscall.ForkLock.RLock() dfd, err := syscall.Dup(int(f.Fd()), -1) - syscall.ForkLock.RUnlock() if err != nil { return nil, os.NewSyscallError("dup", err) } @@ -234,3 +183,7 @@ func setReadBuffer(fd *netFD, bytes int) error { func setWriteBuffer(fd *netFD, bytes int) error { return syscall.EPLAN9 } + +func isHangup(err error) bool { + return err != nil && stringsHasSuffix(err.Error(), "Hangup") +} diff --git a/libgo/go/net/fd_poll_nacl.go b/libgo/go/net/fd_poll_nacl.go index cdf14e3..cda8b82 100644 --- a/libgo/go/net/fd_poll_nacl.go +++ b/libgo/go/net/fd_poll_nacl.go @@ -1,4 +1,4 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -14,44 +14,44 @@ type pollDesc struct { closing bool } -func (pd *pollDesc) Init(fd *netFD) error { pd.fd = fd; return nil } +func (pd *pollDesc) init(fd *netFD) error { pd.fd = fd; return nil } -func (pd *pollDesc) Close() {} +func (pd *pollDesc) close() {} -func (pd *pollDesc) Evict() { +func (pd *pollDesc) evict() { pd.closing = true if pd.fd != nil { syscall.StopIO(pd.fd.sysfd) } } -func (pd *pollDesc) Prepare(mode int) error { +func (pd *pollDesc) prepare(mode int) error { if pd.closing { return errClosing } return nil } -func (pd *pollDesc) PrepareRead() error { return pd.Prepare('r') } +func (pd *pollDesc) prepareRead() error { return pd.prepare('r') } -func (pd *pollDesc) PrepareWrite() error { return pd.Prepare('w') } +func (pd *pollDesc) prepareWrite() error { return pd.prepare('w') } -func (pd *pollDesc) Wait(mode int) error { +func (pd *pollDesc) wait(mode int) error { if pd.closing { return errClosing } return errTimeout } -func (pd *pollDesc) WaitRead() error { return pd.Wait('r') } +func (pd *pollDesc) waitRead() error { return pd.wait('r') } -func (pd *pollDesc) WaitWrite() error { return pd.Wait('w') } +func (pd *pollDesc) waitWrite() error { return pd.wait('w') } -func (pd *pollDesc) WaitCanceled(mode int) {} +func (pd *pollDesc) waitCanceled(mode int) {} -func (pd *pollDesc) WaitCanceledRead() {} +func (pd *pollDesc) waitCanceledRead() {} -func (pd *pollDesc) WaitCanceledWrite() {} +func (pd *pollDesc) waitCanceledWrite() {} func (fd *netFD) setDeadline(t time.Time) error { return setDeadlineImpl(fd, t, 'r'+'w') diff --git a/libgo/go/net/fd_poll_runtime.go b/libgo/go/net/fd_poll_runtime.go index 8522cce..6c1d095 100644 --- a/libgo/go/net/fd_poll_runtime.go +++ b/libgo/go/net/fd_poll_runtime.go @@ -30,7 +30,7 @@ type pollDesc struct { var serverInit sync.Once -func (pd *pollDesc) Init(fd *netFD) error { +func (pd *pollDesc) init(fd *netFD) error { serverInit.Do(runtime_pollServerInit) ctx, errno := runtime_pollOpen(uintptr(fd.sysfd)) if errno != 0 { @@ -40,7 +40,7 @@ func (pd *pollDesc) Init(fd *netFD) error { return nil } -func (pd *pollDesc) Close() { +func (pd *pollDesc) close() { if pd.runtimeCtx == 0 { return } @@ -49,49 +49,49 @@ func (pd *pollDesc) Close() { } // Evict evicts fd from the pending list, unblocking any I/O running on fd. -func (pd *pollDesc) Evict() { +func (pd *pollDesc) evict() { if pd.runtimeCtx == 0 { return } runtime_pollUnblock(pd.runtimeCtx) } -func (pd *pollDesc) Prepare(mode int) error { +func (pd *pollDesc) prepare(mode int) error { res := runtime_pollReset(pd.runtimeCtx, mode) return convertErr(res) } -func (pd *pollDesc) PrepareRead() error { - return pd.Prepare('r') +func (pd *pollDesc) prepareRead() error { + return pd.prepare('r') } -func (pd *pollDesc) PrepareWrite() error { - return pd.Prepare('w') +func (pd *pollDesc) prepareWrite() error { + return pd.prepare('w') } -func (pd *pollDesc) Wait(mode int) error { +func (pd *pollDesc) wait(mode int) error { res := runtime_pollWait(pd.runtimeCtx, mode) return convertErr(res) } -func (pd *pollDesc) WaitRead() error { - return pd.Wait('r') +func (pd *pollDesc) waitRead() error { + return pd.wait('r') } -func (pd *pollDesc) WaitWrite() error { - return pd.Wait('w') +func (pd *pollDesc) waitWrite() error { + return pd.wait('w') } -func (pd *pollDesc) WaitCanceled(mode int) { +func (pd *pollDesc) waitCanceled(mode int) { runtime_pollWaitCanceled(pd.runtimeCtx, mode) } -func (pd *pollDesc) WaitCanceledRead() { - pd.WaitCanceled('r') +func (pd *pollDesc) waitCanceledRead() { + pd.waitCanceled('r') } -func (pd *pollDesc) WaitCanceledWrite() { - pd.WaitCanceled('w') +func (pd *pollDesc) waitCanceledWrite() { + pd.waitCanceled('w') } func convertErr(res int) error { @@ -120,7 +120,13 @@ func (fd *netFD) setWriteDeadline(t time.Time) error { } func setDeadlineImpl(fd *netFD, t time.Time, mode int) error { - d := runtimeNano() + int64(t.Sub(time.Now())) + diff := int64(t.Sub(time.Now())) + d := runtimeNano() + diff + if d <= 0 && diff > 0 { + // If the user has a deadline in the future, but the delay calculation + // overflows, then set the deadline to the maximum possible value. + d = 1<<63 - 1 + } if t.IsZero() { d = 0 } diff --git a/libgo/go/net/fd_unix.go b/libgo/go/net/fd_unix.go index ff498c2..6fbb9cb 100644 --- a/libgo/go/net/fd_unix.go +++ b/libgo/go/net/fd_unix.go @@ -7,12 +7,12 @@ package net import ( + "context" "io" "os" "runtime" "sync/atomic" "syscall" - "time" ) // Network file descriptor. @@ -36,16 +36,12 @@ type netFD struct { func sysInit() { } -func dial(network string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { - return dialer(deadline) -} - func newFD(sysfd, family, sotype int, net string) (*netFD, error) { return &netFD{sysfd: sysfd, family: family, sotype: sotype, net: net}, nil } func (fd *netFD) init() error { - if err := fd.pd.Init(fd); err != nil { + if err := fd.pd.init(fd); err != nil { return err } return nil @@ -68,15 +64,17 @@ func (fd *netFD) name() string { return fd.net + ":" + ls + "->" + rs } -func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-chan struct{}) error { +func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) error { // Do not need to call fd.writeLock here, // because fd is not yet accessible to user, // so no concurrent operations are possible. switch err := connectFunc(fd.sysfd, ra); err { case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR: case nil, syscall.EISCONN: - if !deadline.IsZero() && deadline.Before(time.Now()) { - return errTimeout + select { + case <-ctx.Done(): + return mapErr(ctx.Err()) + default: } if err := fd.init(); err != nil { return err @@ -98,23 +96,27 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c if err := fd.init(); err != nil { return err } - if !deadline.IsZero() { + if deadline, _ := ctx.Deadline(); !deadline.IsZero() { fd.setWriteDeadline(deadline) defer fd.setWriteDeadline(noDeadline) } - if cancel != nil { - done := make(chan bool) - defer close(done) - go func() { - select { - case <-cancel: - // Force the runtime's poller to immediately give - // up waiting for writability. - fd.setWriteDeadline(aLongTimeAgo) - case <-done: - } - }() - } + + // Wait for the goroutine converting context.Done into a write timeout + // to exist, otherwise our caller might cancel the context and + // cause fd.setWriteDeadline(aLongTimeAgo) to cancel a successful dial. + done := make(chan bool) // must be unbuffered + defer func() { done <- true }() + go func() { + select { + case <-ctx.Done(): + // Force the runtime's poller to immediately give + // up waiting for writability. + fd.setWriteDeadline(aLongTimeAgo) + <-done + case <-done: + } + }() + for { // Performing multiple connect system calls on a // non-blocking socket under Unix variants does not @@ -124,10 +126,10 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c // SO_ERROR socket option to see if the connection // succeeded or failed. See issue 7474 for further // details. - if err := fd.pd.WaitWrite(); err != nil { + if err := fd.pd.waitWrite(); err != nil { select { - case <-cancel: - return errCanceled + case <-ctx.Done(): + return mapErr(ctx.Err()) default: } return err @@ -139,7 +141,16 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c switch err := syscall.Errno(nerr); err { case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR: case syscall.Errno(0), syscall.EISCONN: - return nil + if runtime.GOOS != "darwin" { + return nil + } + // See golang.org/issue/14548. + // On Darwin, multiple connect system calls on + // a non-blocking socket never harm SO_ERROR. + switch err := connectFunc(fd.sysfd, ra); err { + case nil, syscall.EISCONN: + return nil + } default: return os.NewSyscallError("getsockopt", err) } @@ -149,71 +160,22 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c func (fd *netFD) destroy() { // Poller may want to unregister fd in readiness notification mechanism, // so this must be executed before closeFunc. - fd.pd.Close() + fd.pd.close() closeFunc(fd.sysfd) fd.sysfd = -1 runtime.SetFinalizer(fd, nil) } -// Add a reference to this fd. -// Returns an error if the fd cannot be used. -func (fd *netFD) incref() error { - if !fd.fdmu.Incref() { - return errClosing - } - return nil -} - -// Remove a reference to this FD and close if we've been asked to do so -// (and there are no references left). -func (fd *netFD) decref() { - if fd.fdmu.Decref() { - fd.destroy() - } -} - -// Add a reference to this fd and lock for reading. -// Returns an error if the fd cannot be used. -func (fd *netFD) readLock() error { - if !fd.fdmu.RWLock(true) { - return errClosing - } - return nil -} - -// Unlock for reading and remove a reference to this FD. -func (fd *netFD) readUnlock() { - if fd.fdmu.RWUnlock(true) { - fd.destroy() - } -} - -// Add a reference to this fd and lock for writing. -// Returns an error if the fd cannot be used. -func (fd *netFD) writeLock() error { - if !fd.fdmu.RWLock(false) { - return errClosing - } - return nil -} - -// Unlock for writing and remove a reference to this FD. -func (fd *netFD) writeUnlock() { - if fd.fdmu.RWUnlock(false) { - fd.destroy() - } -} - func (fd *netFD) Close() error { - if !fd.fdmu.IncrefAndClose() { + if !fd.fdmu.increfAndClose() { return errClosing } // Unblock any I/O. Once it all unblocks and returns, // so that it cannot be referring to fd.sysfd anymore, - // the final decref will close fd.sysfd. This should happen + // the final decref will close fd.sysfd. This should happen // fairly quickly, since all the I/O is non-blocking, and any // attempts to block in the pollDesc will return errClosing. - fd.pd.Evict() + fd.pd.evict() fd.decref() return nil } @@ -239,7 +201,15 @@ func (fd *netFD) Read(p []byte) (n int, err error) { return 0, err } defer fd.readUnlock() - if err := fd.pd.PrepareRead(); err != nil { + if len(p) == 0 { + // If the caller wanted a zero byte read, return immediately + // without trying. (But after acquiring the readLock.) Otherwise + // syscall.Read returns 0, nil and eofError turns that into + // io.EOF. + // TODO(bradfitz): make it wait for readability? (Issue 15735) + return 0, nil + } + if err := fd.pd.prepareRead(); err != nil { return 0, err } for { @@ -247,7 +217,7 @@ func (fd *netFD) Read(p []byte) (n int, err error) { if err != nil { n = 0 if err == syscall.EAGAIN { - if err = fd.pd.WaitRead(); err == nil { + if err = fd.pd.waitRead(); err == nil { continue } } @@ -266,7 +236,7 @@ func (fd *netFD) readFrom(p []byte) (n int, sa syscall.Sockaddr, err error) { return 0, nil, err } defer fd.readUnlock() - if err := fd.pd.PrepareRead(); err != nil { + if err := fd.pd.prepareRead(); err != nil { return 0, nil, err } for { @@ -274,7 +244,7 @@ func (fd *netFD) readFrom(p []byte) (n int, sa syscall.Sockaddr, err error) { if err != nil { n = 0 if err == syscall.EAGAIN { - if err = fd.pd.WaitRead(); err == nil { + if err = fd.pd.waitRead(); err == nil { continue } } @@ -293,7 +263,7 @@ func (fd *netFD) readMsg(p []byte, oob []byte) (n, oobn, flags int, sa syscall.S return 0, 0, 0, nil, err } defer fd.readUnlock() - if err := fd.pd.PrepareRead(); err != nil { + if err := fd.pd.prepareRead(); err != nil { return 0, 0, 0, nil, err } for { @@ -301,7 +271,7 @@ func (fd *netFD) readMsg(p []byte, oob []byte) (n, oobn, flags int, sa syscall.S if err != nil { // TODO(dfc) should n and oobn be set to 0 if err == syscall.EAGAIN { - if err = fd.pd.WaitRead(); err == nil { + if err = fd.pd.waitRead(); err == nil { continue } } @@ -320,7 +290,7 @@ func (fd *netFD) Write(p []byte) (nn int, err error) { return 0, err } defer fd.writeUnlock() - if err := fd.pd.PrepareWrite(); err != nil { + if err := fd.pd.prepareWrite(); err != nil { return 0, err } for { @@ -333,7 +303,7 @@ func (fd *netFD) Write(p []byte) (nn int, err error) { break } if err == syscall.EAGAIN { - if err = fd.pd.WaitWrite(); err == nil { + if err = fd.pd.waitWrite(); err == nil { continue } } @@ -356,13 +326,13 @@ func (fd *netFD) writeTo(p []byte, sa syscall.Sockaddr) (n int, err error) { return 0, err } defer fd.writeUnlock() - if err := fd.pd.PrepareWrite(); err != nil { + if err := fd.pd.prepareWrite(); err != nil { return 0, err } for { err = syscall.Sendto(fd.sysfd, p, 0, sa) if err == syscall.EAGAIN { - if err = fd.pd.WaitWrite(); err == nil { + if err = fd.pd.waitWrite(); err == nil { continue } } @@ -382,13 +352,13 @@ func (fd *netFD) writeMsg(p []byte, oob []byte, sa syscall.Sockaddr) (n int, oob return 0, 0, err } defer fd.writeUnlock() - if err := fd.pd.PrepareWrite(); err != nil { + if err := fd.pd.prepareWrite(); err != nil { return 0, 0, err } for { n, err = syscall.SendmsgN(fd.sysfd, p, oob, sa, 0) if err == syscall.EAGAIN { - if err = fd.pd.WaitWrite(); err == nil { + if err = fd.pd.waitWrite(); err == nil { continue } } @@ -411,7 +381,7 @@ func (fd *netFD) accept() (netfd *netFD, err error) { var s int var rsa syscall.Sockaddr - if err = fd.pd.PrepareRead(); err != nil { + if err = fd.pd.prepareRead(); err != nil { return nil, err } for { @@ -423,7 +393,7 @@ func (fd *netFD) accept() (netfd *netFD, err error) { } switch nerr.Err { case syscall.EAGAIN: - if err = fd.pd.WaitRead(); err == nil { + if err = fd.pd.waitRead(); err == nil { continue } case syscall.ECONNABORTED: @@ -472,7 +442,7 @@ func dupCloseOnExec(fd int) (newfd int, err error) { // and fcntl there falls back (undocumented) // to doing an ioctl instead, returning EBADF // in this case because fd is not of the - // expected device fd type. Treat it as + // expected device fd type. Treat it as // EINVAL instead, so we fall back to the // normal dup path. // TODO: only do this on 10.6 if we can detect 10.6 diff --git a/libgo/go/net/fd_windows.go b/libgo/go/net/fd_windows.go index fd50d77..b0b6769 100644 --- a/libgo/go/net/fd_windows.go +++ b/libgo/go/net/fd_windows.go @@ -5,12 +5,12 @@ package net import ( + "context" "internal/race" "os" "runtime" "sync" "syscall" - "time" "unsafe" ) @@ -42,11 +42,6 @@ func sysInit() { initErr = os.NewSyscallError("wsastartup", e) } canCancelIO = syscall.LoadCancelIoEx() == nil - if syscall.LoadGetAddrInfo() == nil { - lookupPort = newLookupPort - lookupIP = newLookupIP - } - hasLoadSetFileCompletionNotificationModes = syscall.LoadSetFileCompletionNotificationModes() == nil if hasLoadSetFileCompletionNotificationModes { // It's not safe to use FILE_SKIP_COMPLETION_PORT_ON_SUCCESS if non IFS providers are installed: @@ -69,22 +64,15 @@ func sysInit() { } } +// canUseConnectEx reports whether we can use the ConnectEx Windows API call +// for the given network type. func canUseConnectEx(net string) bool { switch net { - case "udp", "udp4", "udp6", "ip", "ip4", "ip6": - // ConnectEx windows API does not support connectionless sockets. - return false - } - return syscall.LoadConnectEx() == nil -} - -func dial(net string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { - if !canUseConnectEx(net) { - // Use the relatively inefficient goroutine-racing - // implementation of DialTimeout. - return dialChannel(net, ra, dialer, deadline) + case "tcp", "tcp4", "tcp6": + return true } - return dialer(deadline) + // ConnectEx windows API does not support connectionless sockets. + return false } // operation contains superset of data necessary to perform all async IO. @@ -152,7 +140,7 @@ func (s *ioSrv) ProcessRemoteIO() { func (s *ioSrv) ExecIO(o *operation, name string, submit func(o *operation) error) (int, error) { fd := o.fd // Notify runtime netpoll about starting IO. - err := fd.pd.Prepare(int(o.mode)) + err := fd.pd.prepare(int(o.mode)) if err != nil { return 0, err } @@ -180,7 +168,7 @@ func (s *ioSrv) ExecIO(o *operation, name string, submit func(o *operation) erro return 0, err } // Wait for our request to complete. - err = fd.pd.Wait(int(o.mode)) + err = fd.pd.wait(int(o.mode)) if err == nil { // All is good. Extract our IO results and return. if o.errno != 0 { @@ -210,7 +198,7 @@ func (s *ioSrv) ExecIO(o *operation, name string, submit func(o *operation) erro <-o.errc } // Wait for cancelation to complete. - fd.pd.WaitCanceled(int(o.mode)) + fd.pd.waitCanceled(int(o.mode)) if o.errno != 0 { err = syscall.Errno(o.errno) if err == syscall.ERROR_OPERATION_ABORTED { // IO Canceled @@ -273,7 +261,7 @@ func newFD(sysfd syscall.Handle, family, sotype int, net string) (*netFD, error) } func (fd *netFD) init() error { - if err := fd.pd.Init(fd); err != nil { + if err := fd.pd.init(fd); err != nil { return err } if hasLoadSetFileCompletionNotificationModes { @@ -320,19 +308,20 @@ func (fd *netFD) setAddr(laddr, raddr Addr) { runtime.SetFinalizer(fd, (*netFD).Close) } -func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-chan struct{}) error { +func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) error { // Do not need to call fd.writeLock here, // because fd is not yet accessible to user, // so no concurrent operations are possible. if err := fd.init(); err != nil { return err } - if !deadline.IsZero() { + if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() { fd.setWriteDeadline(deadline) defer fd.setWriteDeadline(noDeadline) } if !canUseConnectEx(fd.net) { - return os.NewSyscallError("connect", connectFunc(fd.sysfd, ra)) + err := connectFunc(fd.sysfd, ra) + return os.NewSyscallError("connect", err) } // ConnectEx windows API requires an unconnected, previously bound socket. if la == nil { @@ -351,26 +340,30 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c // Call ConnectEx API. o := &fd.wop o.sa = ra - if cancel != nil { - done := make(chan struct{}) - defer close(done) - go func() { - select { - case <-cancel: - // Force the runtime's poller to immediately give - // up waiting for writability. - fd.setWriteDeadline(aLongTimeAgo) - case <-done: - } - }() - } + + // Wait for the goroutine converting context.Done into a write timeout + // to exist, otherwise our caller might cancel the context and + // cause fd.setWriteDeadline(aLongTimeAgo) to cancel a successful dial. + done := make(chan bool) // must be unbuffered + defer func() { done <- true }() + go func() { + select { + case <-ctx.Done(): + // Force the runtime's poller to immediately give + // up waiting for writability. + fd.setWriteDeadline(aLongTimeAgo) + <-done + case <-done: + } + }() + _, err := wsrv.ExecIO(o, "ConnectEx", func(o *operation) error { return connectExFunc(o.fd.sysfd, o.sa, nil, 0, nil, &o.o) }) if err != nil { select { - case <-cancel: - return errCanceled + case <-ctx.Done(): + return mapErr(ctx.Err()) default: if _, ok := err.(syscall.Errno); ok { err = os.NewSyscallError("connectex", err) @@ -388,68 +381,19 @@ func (fd *netFD) destroy() { } // Poller may want to unregister fd in readiness notification mechanism, // so this must be executed before closeFunc. - fd.pd.Close() + fd.pd.close() closeFunc(fd.sysfd) fd.sysfd = syscall.InvalidHandle // no need for a finalizer anymore runtime.SetFinalizer(fd, nil) } -// Add a reference to this fd. -// Returns an error if the fd cannot be used. -func (fd *netFD) incref() error { - if !fd.fdmu.Incref() { - return errClosing - } - return nil -} - -// Remove a reference to this FD and close if we've been asked to do so -// (and there are no references left). -func (fd *netFD) decref() { - if fd.fdmu.Decref() { - fd.destroy() - } -} - -// Add a reference to this fd and lock for reading. -// Returns an error if the fd cannot be used. -func (fd *netFD) readLock() error { - if !fd.fdmu.RWLock(true) { - return errClosing - } - return nil -} - -// Unlock for reading and remove a reference to this FD. -func (fd *netFD) readUnlock() { - if fd.fdmu.RWUnlock(true) { - fd.destroy() - } -} - -// Add a reference to this fd and lock for writing. -// Returns an error if the fd cannot be used. -func (fd *netFD) writeLock() error { - if !fd.fdmu.RWLock(false) { - return errClosing - } - return nil -} - -// Unlock for writing and remove a reference to this FD. -func (fd *netFD) writeUnlock() { - if fd.fdmu.RWUnlock(false) { - fd.destroy() - } -} - func (fd *netFD) Close() error { - if !fd.fdmu.IncrefAndClose() { + if !fd.fdmu.increfAndClose() { return errClosing } // unblock pending reader and writer - fd.pd.Evict() + fd.pd.evict() fd.decref() return nil } @@ -483,7 +427,9 @@ func (fd *netFD) Read(buf []byte) (int, error) { if race.Enabled { race.Acquire(unsafe.Pointer(&ioSync)) } - err = fd.eofError(n, err) + if len(buf) != 0 { + err = fd.eofError(n, err) + } if _, ok := err.(syscall.Errno); ok { err = os.NewSyscallError("wsarecv", err) } @@ -579,7 +525,7 @@ func (fd *netFD) acceptOne(rawsa []syscall.RawSockaddrAny, o *operation) (*netFD o.handle = s o.rsan = int32(unsafe.Sizeof(rawsa[0])) _, err = rsrv.ExecIO(o, "AcceptEx", func(o *operation) error { - return syscall.AcceptEx(o.fd.sysfd, o.handle, (*byte)(unsafe.Pointer(&rawsa[0])), 0, uint32(o.rsan), uint32(o.rsan), &o.qty, &o.o) + return acceptFunc(o.fd.sysfd, o.handle, (*byte)(unsafe.Pointer(&rawsa[0])), 0, uint32(o.rsan), uint32(o.rsan), &o.qty, &o.o) }) if err != nil { netfd.Close() diff --git a/libgo/go/net/file_plan9.go b/libgo/go/net/file_plan9.go index 892775a..2939c09 100644 --- a/libgo/go/net/file_plan9.go +++ b/libgo/go/net/file_plan9.go @@ -50,9 +50,7 @@ func newFileFD(f *os.File) (net *netFD, err error) { name := comp[2] switch file := comp[n-1]; file { case "ctl", "clone": - syscall.ForkLock.RLock() fd, err := syscall.Dup(int(f.Fd()), -1) - syscall.ForkLock.RUnlock() if err != nil { return nil, os.NewSyscallError("dup", err) } @@ -60,7 +58,7 @@ func newFileFD(f *os.File) (net *netFD, err error) { dir := netdir + "/" + comp[n-2] ctl = os.NewFile(uintptr(fd), dir+"/"+file) - ctl.Seek(0, 0) + ctl.Seek(0, io.SeekStart) var buf [16]byte n, err := ctl.Read(buf[:]) if err != nil { diff --git a/libgo/go/net/hook.go b/libgo/go/net/hook.go index 9ab34c0..d7316ea 100644 --- a/libgo/go/net/hook.go +++ b/libgo/go/net/hook.go @@ -4,9 +4,19 @@ package net +import "context" + var ( - testHookDialTCP = dialTCP - testHookHostsPath = "/etc/hosts" - testHookLookupIP = func(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { return fn(host) } + // if non-nil, overrides dialTCP. + testHookDialTCP func(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) + + testHookHostsPath = "/etc/hosts" + testHookLookupIP = func( + ctx context.Context, + fn func(context.Context, string) ([]IPAddr, error), + host string, + ) ([]IPAddr, error) { + return fn(ctx, host) + } testHookSetKeepAlive = func() {} ) diff --git a/libgo/go/net/hook_windows.go b/libgo/go/net/hook_windows.go index 126b0eb..63ea35a 100644 --- a/libgo/go/net/hook_windows.go +++ b/libgo/go/net/hook_windows.go @@ -13,9 +13,10 @@ var ( testHookDialChannel = func() { time.Sleep(time.Millisecond) } // see golang.org/issue/5349 // Placeholders for socket system calls. - socketFunc func(int, int, int) (syscall.Handle, error) = syscall.Socket - closeFunc func(syscall.Handle) error = syscall.Closesocket - connectFunc func(syscall.Handle, syscall.Sockaddr) error = syscall.Connect - connectExFunc func(syscall.Handle, syscall.Sockaddr, *byte, uint32, *uint32, *syscall.Overlapped) error = syscall.ConnectEx - listenFunc func(syscall.Handle, int) error = syscall.Listen + socketFunc func(int, int, int) (syscall.Handle, error) = syscall.Socket + closeFunc func(syscall.Handle) error = syscall.Closesocket + connectFunc func(syscall.Handle, syscall.Sockaddr) error = syscall.Connect + connectExFunc func(syscall.Handle, syscall.Sockaddr, *byte, uint32, *uint32, *syscall.Overlapped) error = syscall.ConnectEx + listenFunc func(syscall.Handle, int) error = syscall.Listen + acceptFunc func(syscall.Handle, syscall.Handle, *byte, uint32, uint32, uint32, *uint32, *syscall.Overlapped) error = syscall.AcceptEx ) diff --git a/libgo/go/net/hosts.go b/libgo/go/net/hosts.go index c4de1b6..9c101c6 100644 --- a/libgo/go/net/hosts.go +++ b/libgo/go/net/hosts.go @@ -110,7 +110,9 @@ func lookupStaticHost(host string) []string { lowerHost := []byte(host) lowerASCIIBytes(lowerHost) if ips, ok := hosts.byName[absDomainName(lowerHost)]; ok { - return ips + ipsCp := make([]string, len(ips)) + copy(ipsCp, ips) + return ipsCp } } return nil @@ -127,7 +129,9 @@ func lookupStaticAddr(addr string) []string { } if len(hosts.byAddr) != 0 { if hosts, ok := hosts.byAddr[addr]; ok { - return hosts + hostsCp := make([]string, len(hosts)) + copy(hostsCp, hosts) + return hostsCp } } return nil diff --git a/libgo/go/net/hosts_test.go b/libgo/go/net/hosts_test.go index 4c67bfa..5d6c9cf 100644 --- a/libgo/go/net/hosts_test.go +++ b/libgo/go/net/hosts_test.go @@ -64,13 +64,17 @@ func TestLookupStaticHost(t *testing.T) { for _, tt := range lookupStaticHostTests { testHookHostsPath = tt.name for _, ent := range tt.ents { - ins := []string{ent.in, absDomainName([]byte(ent.in)), strings.ToLower(ent.in), strings.ToUpper(ent.in)} - for _, in := range ins { - addrs := lookupStaticHost(in) - if !reflect.DeepEqual(addrs, ent.out) { - t.Errorf("%s, lookupStaticHost(%s) = %v; want %v", tt.name, in, addrs, ent.out) - } - } + testStaticHost(t, tt.name, ent) + } + } +} + +func testStaticHost(t *testing.T, hostsPath string, ent staticHostEntry) { + ins := []string{ent.in, absDomainName([]byte(ent.in)), strings.ToLower(ent.in), strings.ToUpper(ent.in)} + for _, in := range ins { + addrs := lookupStaticHost(in) + if !reflect.DeepEqual(addrs, ent.out) { + t.Errorf("%s, lookupStaticHost(%s) = %v; want %v", hostsPath, in, addrs, ent.out) } } } @@ -129,13 +133,43 @@ func TestLookupStaticAddr(t *testing.T) { for _, tt := range lookupStaticAddrTests { testHookHostsPath = tt.name for _, ent := range tt.ents { - hosts := lookupStaticAddr(ent.in) - for i := range ent.out { - ent.out[i] = absDomainName([]byte(ent.out[i])) - } - if !reflect.DeepEqual(hosts, ent.out) { - t.Errorf("%s, lookupStaticAddr(%s) = %v; want %v", tt.name, ent.in, hosts, ent.out) - } + testStaticAddr(t, tt.name, ent) } } } + +func testStaticAddr(t *testing.T, hostsPath string, ent staticHostEntry) { + hosts := lookupStaticAddr(ent.in) + for i := range ent.out { + ent.out[i] = absDomainName([]byte(ent.out[i])) + } + if !reflect.DeepEqual(hosts, ent.out) { + t.Errorf("%s, lookupStaticAddr(%s) = %v; want %v", hostsPath, ent.in, hosts, ent.out) + } +} + +func TestHostCacheModification(t *testing.T) { + // Ensure that programs can't modify the internals of the host cache. + // See https://github.com/golang/go/issues/14212. + defer func(orig string) { testHookHostsPath = orig }(testHookHostsPath) + + testHookHostsPath = "testdata/ipv4-hosts" + ent := staticHostEntry{"localhost", []string{"127.0.0.1", "127.0.0.2", "127.0.0.3"}} + testStaticHost(t, testHookHostsPath, ent) + // Modify the addresses return by lookupStaticHost. + addrs := lookupStaticHost(ent.in) + for i := range addrs { + addrs[i] += "junk" + } + testStaticHost(t, testHookHostsPath, ent) + + testHookHostsPath = "testdata/ipv6-hosts" + ent = staticHostEntry{"::1", []string{"localhost"}} + testStaticAddr(t, testHookHostsPath, ent) + // Modify the hosts return by lookupStaticAddr. + hosts := lookupStaticAddr(ent.in) + for i := range hosts { + hosts[i] += "junk" + } + testStaticAddr(t, testHookHostsPath, ent) +} diff --git a/libgo/go/net/http/cgi/host.go b/libgo/go/net/http/cgi/host.go index 9b4d875..58e9f71 100644 --- a/libgo/go/net/http/cgi/host.go +++ b/libgo/go/net/http/cgi/host.go @@ -10,7 +10,7 @@ // // Note that using CGI means starting a new process to handle each // request, which is typically less efficient than using a -// long-running server. This package is intended primarily for +// long-running server. This package is intended primarily for // compatibility with existing systems. package cgi @@ -58,6 +58,7 @@ type Handler struct { InheritEnv []string // environment variables to inherit from host, as "key" Logger *log.Logger // optional log for errors or nil to use log.Print Args []string // optional arguments to pass to child process + Stderr io.Writer // optional stderr for the child process; nil means os.Stderr // PathLocationHandler specifies the root http Handler that // should handle internal redirects when the CGI process @@ -70,6 +71,13 @@ type Handler struct { PathLocationHandler http.Handler } +func (h *Handler) stderr() io.Writer { + if h.Stderr != nil { + return h.Stderr + } + return os.Stderr +} + // removeLeadingDuplicates remove leading duplicate in environments. // It's possible to override environment like following. // cgi.Handler{ @@ -145,6 +153,10 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { for k, v := range req.Header { k = strings.Map(upperCaseAndUnderscore, k) + if k == "PROXY" { + // See Issue 16405 + continue + } joinStr := ", " if k == "COOKIE" { joinStr = "; " @@ -204,7 +216,7 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { Args: append([]string{h.Path}, h.Args...), Dir: cwd, Env: env, - Stderr: os.Stderr, // for now + Stderr: h.stderr(), } if req.ContentLength != 0 { cmd.Stdin = req.Body diff --git a/libgo/go/net/http/cgi/host_test.go b/libgo/go/net/http/cgi/host_test.go index fb7d66a..f058372 100644 --- a/libgo/go/net/http/cgi/host_test.go +++ b/libgo/go/net/http/cgi/host_test.go @@ -8,6 +8,7 @@ package cgi import ( "bufio" + "bytes" "fmt" "io" "net" @@ -34,15 +35,18 @@ func newRequest(httpreq string) *http.Request { return req } -func runCgiTest(t *testing.T, h *Handler, httpreq string, expectedMap map[string]string) *httptest.ResponseRecorder { +func runCgiTest(t *testing.T, h *Handler, + httpreq string, + expectedMap map[string]string, checks ...func(reqInfo map[string]string)) *httptest.ResponseRecorder { rw := httptest.NewRecorder() req := newRequest(httpreq) h.ServeHTTP(rw, req) - runResponseChecks(t, rw, expectedMap) + runResponseChecks(t, rw, expectedMap, checks...) return rw } -func runResponseChecks(t *testing.T, rw *httptest.ResponseRecorder, expectedMap map[string]string) { +func runResponseChecks(t *testing.T, rw *httptest.ResponseRecorder, + expectedMap map[string]string, checks ...func(reqInfo map[string]string)) { // Make a map to hold the test map that the CGI returns. m := make(map[string]string) m["_body"] = rw.Body.String() @@ -80,6 +84,9 @@ readlines: t.Errorf("for key %q got %q; expected %q", key, got, expected) } } + for _, check := range checks { + check(m) + } } var cgiTested, cgiWorks bool @@ -235,6 +242,31 @@ func TestDupHeaders(t *testing.T) { expectedMap) } +// Issue 16405: CGI+http.Transport differing uses of HTTP_PROXY. +// Verify we don't set the HTTP_PROXY environment variable. +// Hope nobody was depending on it. It's not a known header, though. +func TestDropProxyHeader(t *testing.T) { + check(t) + h := &Handler{ + Path: "testdata/test.cgi", + } + expectedMap := map[string]string{ + "env-REQUEST_URI": "/myscript/bar?a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-HTTP_X_FOO": "a", + } + runCgiTest(t, h, "GET /myscript/bar?a=b HTTP/1.0\n"+ + "X-Foo: a\n"+ + "Proxy: should_be_stripped\n"+ + "Host: example.com\n\n", + expectedMap, + func(reqInfo map[string]string) { + if v, ok := reqInfo["env-HTTP_PROXY"]; ok { + t.Errorf("HTTP_PROXY = %q; should be absent", v) + } + }) +} + func TestPathInfoNoRoot(t *testing.T) { check(t) h := &Handler{ @@ -501,6 +533,23 @@ func TestEnvOverride(t *testing.T) { runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) } +func TestHandlerStderr(t *testing.T) { + check(t) + var stderr bytes.Buffer + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + Stderr: &stderr, + } + + rw := httptest.NewRecorder() + req := newRequest("GET /test.cgi?writestderr=1 HTTP/1.0\nHost: example.com\n\n") + h.ServeHTTP(rw, req) + if got, want := stderr.String(), "Hello, stderr!\n"; got != want { + t.Errorf("Stderr = %q; want %q", got, want) + } +} + func TestRemoveLeadingDuplicates(t *testing.T) { tests := []struct { env []string diff --git a/libgo/go/net/http/cgi/testdata/test.cgi b/libgo/go/net/http/cgi/testdata/test.cgi index ec7ee6f..667fce2 100644 --- a/libgo/go/net/http/cgi/testdata/test.cgi +++ b/libgo/go/net/http/cgi/testdata/test.cgi @@ -23,6 +23,10 @@ print "X-CGI-Pid: $$\r\n"; print "X-Test-Header: X-Test-Value\r\n"; print "\r\n"; +if ($params->{"writestderr"}) { + print STDERR "Hello, stderr!\n"; +} + if ($params->{"bigresponse"}) { # 17 MB, for OS X: golang.org/issue/4958 for (1..(17 * 1024)) { diff --git a/libgo/go/net/http/client.go b/libgo/go/net/http/client.go index 3106d22..993c247 100644 --- a/libgo/go/net/http/client.go +++ b/libgo/go/net/http/client.go @@ -44,9 +44,12 @@ type Client struct { // following an HTTP redirect. The arguments req and via are // the upcoming request and the requests made already, oldest // first. If CheckRedirect returns an error, the Client's Get - // method returns both the previous Response and - // CheckRedirect's error (wrapped in a url.Error) instead of - // issuing the Request req. + // method returns both the previous Response (with its Body + // closed) and CheckRedirect's error (wrapped in a url.Error) + // instead of issuing the Request req. + // As a special case, if CheckRedirect returns ErrUseLastResponse, + // then the most recent response is returned with its body + // unclosed, along with a nil error. // // If CheckRedirect is nil, the Client uses its default policy, // which is to stop after 10 consecutive requests. @@ -110,10 +113,6 @@ type RoundTripper interface { RoundTrip(*Request) (*Response, error) } -// Given a string of the form "host", "host:port", or "[ipv6::address]:port", -// return true if the string includes a port. -func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } - // refererForURL returns a referer without any authentication info or // an empty string if lastReq scheme is https and newReq scheme is http. func refererForURL(lastReq, newReq *url.URL) string { @@ -138,14 +137,6 @@ func refererForURL(lastReq, newReq *url.URL) string { return referer } -// Used in Send to implement io.ReadCloser by bundling together the -// bufio.Reader through which we read the response, and the underlying -// network connection. -type readClose struct { - io.Reader - io.Closer -} - func (c *Client) send(req *Request, deadline time.Time) (*Response, error) { if c.Jar != nil { for _, cookie := range c.Jar.Cookies(req.URL) { @@ -161,28 +152,33 @@ func (c *Client) send(req *Request, deadline time.Time) (*Response, error) { c.Jar.SetCookies(req.URL, rc) } } - return resp, err + return resp, nil } // Do sends an HTTP request and returns an HTTP response, following -// policy (e.g. redirects, cookies, auth) as configured on the client. +// policy (such as redirects, cookies, auth) as configured on the +// client. // // An error is returned if caused by client policy (such as -// CheckRedirect), or if there was an HTTP protocol error. -// A non-2xx response doesn't cause an error. -// -// When err is nil, resp always contains a non-nil resp.Body. +// CheckRedirect), or failure to speak HTTP (such as a network +// connectivity problem). A non-2xx status code doesn't cause an +// error. // -// Callers should close resp.Body when done reading from it. If -// resp.Body is not closed, the Client's underlying RoundTripper -// (typically Transport) may not be able to re-use a persistent TCP -// connection to the server for a subsequent "keep-alive" request. +// If the returned error is nil, the Response will contain a non-nil +// Body which the user is expected to close. If the Body is not +// closed, the Client's underlying RoundTripper (typically Transport) +// may not be able to re-use a persistent TCP connection to the server +// for a subsequent "keep-alive" request. // // The request Body, if non-nil, will be closed by the underlying // Transport, even on errors. // +// On error, any Response can be ignored. A non-nil Response with a +// non-nil error only occurs when CheckRedirect fails, and even then +// the returned Response.Body is already closed. +// // Generally Get, Post, or PostForm will be used instead of Do. -func (c *Client) Do(req *Request) (resp *Response, err error) { +func (c *Client) Do(req *Request) (*Response, error) { method := valueOrDefault(req.Method, "GET") if method == "GET" || method == "HEAD" { return c.doFollowingRedirects(req, shouldRedirectGet) @@ -237,7 +233,7 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) } // Most the callers of send (Get, Post, et al) don't need - // Headers, leaving it uninitialized. We guarantee to the + // Headers, leaving it uninitialized. We guarantee to the // Transport that this has been initialized, though. if req.Header == nil { forkReq() @@ -424,101 +420,125 @@ func (c *Client) Get(url string) (resp *Response, err error) { func alwaysFalse() bool { return false } -func (c *Client) doFollowingRedirects(ireq *Request, shouldRedirect func(int) bool) (resp *Response, err error) { - var base *url.URL - redirectChecker := c.CheckRedirect - if redirectChecker == nil { - redirectChecker = defaultCheckRedirect +// ErrUseLastResponse can be returned by Client.CheckRedirect hooks to +// control how redirects are processed. If returned, the next request +// is not sent and the most recent response is returned with its body +// unclosed. +var ErrUseLastResponse = errors.New("net/http: use last response") + +// checkRedirect calls either the user's configured CheckRedirect +// function, or the default. +func (c *Client) checkRedirect(req *Request, via []*Request) error { + fn := c.CheckRedirect + if fn == nil { + fn = defaultCheckRedirect } - var via []*Request + return fn(req, via) +} - if ireq.URL == nil { - ireq.closeBody() +func (c *Client) doFollowingRedirects(req *Request, shouldRedirect func(int) bool) (*Response, error) { + if req.URL == nil { + req.closeBody() return nil, errors.New("http: nil Request.URL") } - req := ireq - deadline := c.deadline() - - urlStr := "" // next relative or absolute URL to fetch (after first request) - redirectFailed := false - for redirect := 0; ; redirect++ { - if redirect != 0 { - nreq := new(Request) - nreq.Cancel = ireq.Cancel - nreq.Method = ireq.Method - if ireq.Method == "POST" || ireq.Method == "PUT" { - nreq.Method = "GET" + var ( + deadline = c.deadline() + reqs []*Request + resp *Response + ) + uerr := func(err error) error { + req.closeBody() + method := valueOrDefault(reqs[0].Method, "GET") + var urlStr string + if resp != nil && resp.Request != nil { + urlStr = resp.Request.URL.String() + } else { + urlStr = req.URL.String() + } + return &url.Error{ + Op: method[:1] + strings.ToLower(method[1:]), + URL: urlStr, + Err: err, + } + } + for { + // For all but the first request, create the next + // request hop and replace req. + if len(reqs) > 0 { + loc := resp.Header.Get("Location") + if loc == "" { + return nil, uerr(fmt.Errorf("%d response missing Location header", resp.StatusCode)) } - nreq.Header = make(Header) - nreq.URL, err = base.Parse(urlStr) + u, err := req.URL.Parse(loc) if err != nil { - break + return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err)) } - if len(via) > 0 { - // Add the Referer header. - lastReq := via[len(via)-1] - if ref := refererForURL(lastReq.URL, nreq.URL); ref != "" { - nreq.Header.Set("Referer", ref) - } - - err = redirectChecker(nreq, via) - if err != nil { - redirectFailed = true - break - } + ireq := reqs[0] + req = &Request{ + Method: ireq.Method, + Response: resp, + URL: u, + Header: make(Header), + Cancel: ireq.Cancel, + ctx: ireq.ctx, } - req = nreq - } + if ireq.Method == "POST" || ireq.Method == "PUT" { + req.Method = "GET" + } + // Add the Referer header from the most recent + // request URL to the new one, if it's not https->http: + if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" { + req.Header.Set("Referer", ref) + } + err = c.checkRedirect(req, reqs) - urlStr = req.URL.String() - if resp, err = c.send(req, deadline); err != nil { - if !deadline.IsZero() && !time.Now().Before(deadline) { - err = &httpError{ - err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", - timeout: true, - } + // Sentinel error to let users select the + // previous response, without closing its + // body. See Issue 10069. + if err == ErrUseLastResponse { + return resp, nil } - break - } - if shouldRedirect(resp.StatusCode) { - // Read the body if small so underlying TCP connection will be re-used. - // No need to check for errors: if it fails, Transport won't reuse it anyway. + // Close the previous response's body. But + // read at least some of the body so if it's + // small the underlying TCP connection will be + // re-used. No need to check for errors: if it + // fails, the Transport won't reuse it anyway. const maxBodySlurpSize = 2 << 10 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize) } resp.Body.Close() - if urlStr = resp.Header.Get("Location"); urlStr == "" { - err = fmt.Errorf("%d response missing Location header", resp.StatusCode) - break + + if err != nil { + // Special case for Go 1 compatibility: return both the response + // and an error if the CheckRedirect function failed. + // See https://golang.org/issue/3795 + // The resp.Body has already been closed. + ue := uerr(err) + ue.(*url.Error).URL = loc + return resp, ue } - base = req.URL - via = append(via, req) - continue } - return resp, nil - } - method := valueOrDefault(ireq.Method, "GET") - urlErr := &url.Error{ - Op: method[:1] + strings.ToLower(method[1:]), - URL: urlStr, - Err: err, - } + reqs = append(reqs, req) - if redirectFailed { - // Special case for Go 1 compatibility: return both the response - // and an error if the CheckRedirect function failed. - // See https://golang.org/issue/3795 - return resp, urlErr - } + var err error + if resp, err = c.send(req, deadline); err != nil { + if !deadline.IsZero() && !time.Now().Before(deadline) { + err = &httpError{ + err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", + timeout: true, + } + } + return nil, uerr(err) + } - if resp != nil { - resp.Body.Close() + if !shouldRedirect(resp.StatusCode) { + return resp, nil + } } - return nil, urlErr } func defaultCheckRedirect(req *Request, via []*Request) error { diff --git a/libgo/go/net/http/client_test.go b/libgo/go/net/http/client_test.go index 8939dc8..a9b1948 100644 --- a/libgo/go/net/http/client_test.go +++ b/libgo/go/net/http/client_test.go @@ -8,6 +8,7 @@ package http_test import ( "bytes" + "context" "crypto/tls" "crypto/x509" "encoding/base64" @@ -273,7 +274,7 @@ func TestClientRedirects(t *testing.T) { t.Fatal("didn't see redirect") } if lastReq.Cancel != cancel { - t.Errorf("expected lastReq to have the cancel channel set on the inital req") + t.Errorf("expected lastReq to have the cancel channel set on the initial req") } checkErr = errors.New("no redirects allowed") @@ -290,6 +291,33 @@ func TestClientRedirects(t *testing.T) { } } +func TestClientRedirectContext(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + Redirect(w, r, "/", StatusFound) + })) + defer ts.Close() + + ctx, cancel := context.WithCancel(context.Background()) + c := &Client{CheckRedirect: func(req *Request, via []*Request) error { + cancel() + if len(via) > 2 { + return errors.New("too many redirects") + } + return nil + }} + req, _ := NewRequest("GET", ts.URL, nil) + req = req.WithContext(ctx) + _, err := c.Do(req) + ue, ok := err.(*url.Error) + if !ok { + t.Fatalf("got error %T; want *url.Error", err) + } + if ue.Err != ExportErrRequestCanceled && ue.Err != ExportErrRequestCanceledConn { + t.Errorf("url.Error.Err = %v; want errRequestCanceled or errRequestCanceledConn", ue.Err) + } +} + func TestPostRedirects(t *testing.T) { defer afterTest(t) var log struct { @@ -338,6 +366,44 @@ func TestPostRedirects(t *testing.T) { } } +func TestClientRedirectUseResponse(t *testing.T) { + defer afterTest(t) + const body = "Hello, world." + var ts *httptest.Server + ts = httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if strings.Contains(r.URL.Path, "/other") { + io.WriteString(w, "wrong body") + } else { + w.Header().Set("Location", ts.URL+"/other") + w.WriteHeader(StatusFound) + io.WriteString(w, body) + } + })) + defer ts.Close() + + c := &Client{CheckRedirect: func(req *Request, via []*Request) error { + if req.Response == nil { + t.Error("expected non-nil Request.Response") + } + return ErrUseLastResponse + }} + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != StatusFound { + t.Errorf("status = %d; want %d", res.StatusCode, StatusFound) + } + defer res.Body.Close() + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if string(slurp) != body { + t.Errorf("body = %q; want %q", slurp, body) + } +} + var expectedCookies = []*Cookie{ {Name: "ChocolateChip", Value: "tasty"}, {Name: "First", Value: "Hit"}, @@ -1140,3 +1206,26 @@ func TestReferer(t *testing.T) { } } } + +// issue15577Tripper returns a Response with a redirect response +// header and doesn't populate its Response.Request field. +type issue15577Tripper struct{} + +func (issue15577Tripper) RoundTrip(*Request) (*Response, error) { + resp := &Response{ + StatusCode: 303, + Header: map[string][]string{"Location": {"http://www.example.com/"}}, + Body: ioutil.NopCloser(strings.NewReader("")), + } + return resp, nil +} + +// Issue 15577: don't assume the roundtripper's response populates its Request field. +func TestClientRedirectResponseWithoutRequest(t *testing.T) { + c := &Client{ + CheckRedirect: func(*Request, []*Request) error { return fmt.Errorf("no redirects!") }, + Transport: issue15577Tripper{}, + } + // Check that this doesn't crash: + c.Get("http://dummy.tld") +} diff --git a/libgo/go/net/http/clientserver_test.go b/libgo/go/net/http/clientserver_test.go index aa2473a..3d1f09c 100644 --- a/libgo/go/net/http/clientserver_test.go +++ b/libgo/go/net/http/clientserver_test.go @@ -17,6 +17,7 @@ import ( "net" . "net/http" "net/http/httptest" + "net/http/httputil" "net/url" "os" "reflect" @@ -43,6 +44,13 @@ func (t *clientServerTest) close() { t.ts.Close() } +func (t *clientServerTest) scheme() string { + if t.h2 { + return "https" + } + return "http" +} + const ( h1Mode = false h2Mode = true @@ -147,10 +155,11 @@ type reqFunc func(c *Client, url string) (*Response, error) // h12Compare is a test that compares HTTP/1 and HTTP/2 behavior // against each other. type h12Compare struct { - Handler func(ResponseWriter, *Request) // required - ReqFunc reqFunc // optional - CheckResponse func(proto string, res *Response) // optional - Opts []interface{} + Handler func(ResponseWriter, *Request) // required + ReqFunc reqFunc // optional + CheckResponse func(proto string, res *Response) // optional + EarlyCheckResponse func(proto string, res *Response) // optional; pre-normalize + Opts []interface{} } func (tt h12Compare) reqFunc() reqFunc { @@ -176,6 +185,12 @@ func (tt h12Compare) run(t *testing.T) { t.Errorf("HTTP/2 request: %v", err) return } + + if fn := tt.EarlyCheckResponse; fn != nil { + fn("HTTP/1.1", res1) + fn("HTTP/2.0", res2) + } + tt.normalizeRes(t, res1, "HTTP/1.1") tt.normalizeRes(t, res2, "HTTP/2.0") res1body, res2body := res1.Body, res2.Body @@ -220,6 +235,7 @@ func (tt h12Compare) normalizeRes(t *testing.T, res *Response, wantProto string) t.Errorf("got %q response; want %q", res.Proto, wantProto) } slurp, err := ioutil.ReadAll(res.Body) + res.Body.Close() res.Body = slurpResult{ ReadCloser: ioutil.NopCloser(bytes.NewReader(slurp)), @@ -356,7 +372,7 @@ func TestH12_HandlerWritesTooLittle(t *testing.T) { } // Tests that the HTTP/1 and HTTP/2 servers prevent handlers from -// writing more than they declared. This test does not test whether +// writing more than they declared. This test does not test whether // the transport deals with too much data, though, since the server // doesn't make it possible to send bogus data. For those tests, see // transport_test.go (for HTTP/1) or x/net/http2/transport_test.go @@ -1049,6 +1065,170 @@ func testTransportGCRequest(t *testing.T, h2, body bool) { } } +func TestTransportRejectsInvalidHeaders_h1(t *testing.T) { + testTransportRejectsInvalidHeaders(t, h1Mode) +} +func TestTransportRejectsInvalidHeaders_h2(t *testing.T) { + testTransportRejectsInvalidHeaders(t, h2Mode) +} +func testTransportRejectsInvalidHeaders(t *testing.T, h2 bool) { + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "Handler saw headers: %q", r.Header) + })) + defer cst.close() + cst.tr.DisableKeepAlives = true + + tests := []struct { + key, val string + ok bool + }{ + {"Foo", "capital-key", true}, // verify h2 allows capital keys + {"Foo", "foo\x00bar", false}, // \x00 byte in value not allowed + {"Foo", "two\nlines", false}, // \n byte in value not allowed + {"bogus\nkey", "v", false}, // \n byte also not allowed in key + {"A space", "v", false}, // spaces in keys not allowed + {"имя", "v", false}, // key must be ascii + {"name", "валю", true}, // value may be non-ascii + {"", "v", false}, // key must be non-empty + {"k", "", true}, // value may be empty + } + for _, tt := range tests { + dialedc := make(chan bool, 1) + cst.tr.Dial = func(netw, addr string) (net.Conn, error) { + dialedc <- true + return net.Dial(netw, addr) + } + req, _ := NewRequest("GET", cst.ts.URL, nil) + req.Header[tt.key] = []string{tt.val} + res, err := cst.c.Do(req) + var body []byte + if err == nil { + body, _ = ioutil.ReadAll(res.Body) + res.Body.Close() + } + var dialed bool + select { + case <-dialedc: + dialed = true + default: + } + + if !tt.ok && dialed { + t.Errorf("For key %q, value %q, transport dialed. Expected local failure. Response was: (%v, %v)\nServer replied with: %s", tt.key, tt.val, res, err, body) + } else if (err == nil) != tt.ok { + t.Errorf("For key %q, value %q; got err = %v; want ok=%v", tt.key, tt.val, err, tt.ok) + } + } +} + +// Tests that we support bogus under-100 HTTP statuses, because we historically +// have. This might change at some point, but not yet in Go 1.6. +func TestBogusStatusWorks_h1(t *testing.T) { testBogusStatusWorks(t, h1Mode) } +func TestBogusStatusWorks_h2(t *testing.T) { testBogusStatusWorks(t, h2Mode) } +func testBogusStatusWorks(t *testing.T, h2 bool) { + defer afterTest(t) + const code = 7 + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + w.WriteHeader(code) + })) + defer cst.close() + + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != code { + t.Errorf("StatusCode = %d; want %d", res.StatusCode, code) + } +} + +func TestInterruptWithPanic_h1(t *testing.T) { testInterruptWithPanic(t, h1Mode) } +func TestInterruptWithPanic_h2(t *testing.T) { testInterruptWithPanic(t, h2Mode) } +func testInterruptWithPanic(t *testing.T, h2 bool) { + log.SetOutput(ioutil.Discard) // is noisy otherwise + defer log.SetOutput(os.Stderr) + + const msg = "hello" + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + io.WriteString(w, msg) + w.(Flusher).Flush() + panic("no more") + })) + defer cst.close() + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + slurp, err := ioutil.ReadAll(res.Body) + if string(slurp) != msg { + t.Errorf("client read %q; want %q", slurp, msg) + } + if err == nil { + t.Errorf("client read all successfully; want some error") + } +} + +// Issue 15366 +func TestH12_AutoGzipWithDumpResponse(t *testing.T) { + h12Compare{ + Handler: func(w ResponseWriter, r *Request) { + h := w.Header() + h.Set("Content-Encoding", "gzip") + h.Set("Content-Length", "23") + h.Set("Connection", "keep-alive") + io.WriteString(w, "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00") + }, + EarlyCheckResponse: func(proto string, res *Response) { + if !res.Uncompressed { + t.Errorf("%s: expected Uncompressed to be set", proto) + } + dump, err := httputil.DumpResponse(res, true) + if err != nil { + t.Errorf("%s: DumpResponse: %v", proto, err) + return + } + if strings.Contains(string(dump), "Connection: close") { + t.Errorf("%s: should not see \"Connection: close\" in dump; got:\n%s", proto, dump) + } + if !strings.Contains(string(dump), "FOO") { + t.Errorf("%s: should see \"FOO\" in response; got:\n%s", proto, dump) + } + }, + }.run(t) +} + +// Issue 14607 +func TestCloseIdleConnections_h1(t *testing.T) { testCloseIdleConnections(t, h1Mode) } +func TestCloseIdleConnections_h2(t *testing.T) { testCloseIdleConnections(t, h2Mode) } +func testCloseIdleConnections(t *testing.T, h2 bool) { + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("X-Addr", r.RemoteAddr) + })) + defer cst.close() + get := func() string { + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + v := res.Header.Get("X-Addr") + if v == "" { + t.Fatal("didn't get X-Addr") + } + return v + } + a1 := get() + cst.tr.CloseIdleConnections() + a2 := get() + if a1 == a2 { + t.Errorf("didn't close connection") + } +} + type noteCloseConn struct { net.Conn closeFunc func() diff --git a/libgo/go/net/http/cookie.go b/libgo/go/net/http/cookie.go index 648709d..1ea0e93 100644 --- a/libgo/go/net/http/cookie.go +++ b/libgo/go/net/http/cookie.go @@ -223,7 +223,7 @@ func readCookies(h Header, filter string) []*Cookie { return cookies } -// validCookieDomain returns wheter v is a valid cookie domain-value. +// validCookieDomain returns whether v is a valid cookie domain-value. func validCookieDomain(v string) bool { if isCookieDomainName(v) { return true diff --git a/libgo/go/net/http/cookie_test.go b/libgo/go/net/http/cookie_test.go index d474f31..95e6147 100644 --- a/libgo/go/net/http/cookie_test.go +++ b/libgo/go/net/http/cookie_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/http/cookiejar/punycode.go b/libgo/go/net/http/cookiejar/punycode.go index ea7ceb5..a9cc666 100644 --- a/libgo/go/net/http/cookiejar/punycode.go +++ b/libgo/go/net/http/cookiejar/punycode.go @@ -37,7 +37,7 @@ func encode(prefix, s string) (string, error) { delta, n, bias := int32(0), initialN, initialBias b, remaining := int32(0), int32(0) for _, r := range s { - if r < 0x80 { + if r < utf8.RuneSelf { b++ output = append(output, byte(r)) } else { diff --git a/libgo/go/net/http/export_test.go b/libgo/go/net/http/export_test.go index 52bccbd..9c5ba08 100644 --- a/libgo/go/net/http/export_test.go +++ b/libgo/go/net/http/export_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -9,22 +9,22 @@ package http import ( "net" + "sort" "sync" "time" ) var ( - DefaultUserAgent = defaultUserAgent - NewLoggingConn = newLoggingConn - ExportAppendTime = appendTime - ExportRefererForURL = refererForURL - ExportServerNewConn = (*Server).newConn - ExportCloseWriteAndWait = (*conn).closeWriteAndWait - ExportErrRequestCanceled = errRequestCanceled - ExportErrRequestCanceledConn = errRequestCanceledConn - ExportServeFile = serveFile - ExportHttp2ConfigureTransport = http2ConfigureTransport - ExportHttp2ConfigureServer = http2ConfigureServer + DefaultUserAgent = defaultUserAgent + NewLoggingConn = newLoggingConn + ExportAppendTime = appendTime + ExportRefererForURL = refererForURL + ExportServerNewConn = (*Server).newConn + ExportCloseWriteAndWait = (*conn).closeWriteAndWait + ExportErrRequestCanceled = errRequestCanceled + ExportErrRequestCanceledConn = errRequestCanceledConn + ExportServeFile = serveFile + ExportHttp2ConfigureServer = http2ConfigureServer ) func init() { @@ -35,9 +35,8 @@ func init() { } var ( - SetEnterRoundTripHook = hookSetter(&testHookEnterRoundTrip) - SetTestHookWaitResLoop = hookSetter(&testHookWaitResLoop) - SetRoundTripRetried = hookSetter(&testHookRoundTripRetried) + SetEnterRoundTripHook = hookSetter(&testHookEnterRoundTrip) + SetRoundTripRetried = hookSetter(&testHookRoundTripRetried) ) func SetReadLoopBeforeNextReadHook(f func()) { @@ -59,9 +58,9 @@ func SetTestHookServerServe(fn func(*Server, net.Listener)) { testHookServerServ func NewTestTimeoutHandler(handler Handler, ch <-chan time.Time) Handler { return &timeoutHandler{ - handler: handler, - timeout: func() <-chan time.Time { return ch }, - // (no body and nil cancelTimer) + handler: handler, + testTimeout: ch, + // (no body) } } @@ -81,21 +80,29 @@ func (t *Transport) IdleConnKeysForTesting() (keys []string) { keys = make([]string, 0) t.idleMu.Lock() defer t.idleMu.Unlock() - if t.idleConn == nil { - return - } for key := range t.idleConn { keys = append(keys, key.String()) } + sort.Strings(keys) return } -func (t *Transport) IdleConnCountForTesting(cacheKey string) int { +func (t *Transport) IdleConnStrsForTesting() []string { + var ret []string t.idleMu.Lock() defer t.idleMu.Unlock() - if t.idleConn == nil { - return 0 + for _, conns := range t.idleConn { + for _, pc := range conns { + ret = append(ret, pc.conn.LocalAddr().String()+"/"+pc.conn.RemoteAddr().String()) + } } + sort.Strings(ret) + return ret +} + +func (t *Transport) IdleConnCountForTesting(cacheKey string) int { + t.idleMu.Lock() + defer t.idleMu.Unlock() for k, conns := range t.idleConn { if k.String() == cacheKey { return len(conns) @@ -144,3 +151,12 @@ func hookSetter(dst *func()) func(func()) { *dst = fn } } + +func ExportHttp2ConfigureTransport(t *Transport) error { + t2, err := http2configureTransport(t) + if err != nil { + return err + } + t.h2transport = t2 + return nil +} diff --git a/libgo/go/net/http/fcgi/fcgi.go b/libgo/go/net/http/fcgi/fcgi.go index 06bba04..3374841 100644 --- a/libgo/go/net/http/fcgi/fcgi.go +++ b/libgo/go/net/http/fcgi/fcgi.go @@ -58,8 +58,6 @@ const ( statusUnknownRole ) -const headerLen = 8 - type header struct { Version uint8 Type recType @@ -158,11 +156,6 @@ func (c *conn) writeRecord(recType recType, reqId uint16, b []byte) error { return err } -func (c *conn) writeBeginRequest(reqId uint16, role uint16, flags uint8) error { - b := [8]byte{byte(role >> 8), byte(role), flags} - return c.writeRecord(typeBeginRequest, reqId, b[:]) -} - func (c *conn) writeEndRequest(reqId uint16, appStatus int, protocolStatus uint8) error { b := make([]byte, 8) binary.BigEndian.PutUint32(b, uint32(appStatus)) diff --git a/libgo/go/net/http/filetransport.go b/libgo/go/net/http/filetransport.go index 821787e..32126d7 100644 --- a/libgo/go/net/http/filetransport.go +++ b/libgo/go/net/http/filetransport.go @@ -33,7 +33,7 @@ func NewFileTransport(fs FileSystem) RoundTripper { func (t fileTransport) RoundTrip(req *Request) (resp *Response, err error) { // We start ServeHTTP in a goroutine, which may take a long - // time if the file is large. The newPopulateResponseWriter + // time if the file is large. The newPopulateResponseWriter // call returns a channel which either ServeHTTP or finish() // sends our *Response on, once the *Response itself has been // populated (even if the body itself is still being diff --git a/libgo/go/net/http/fs.go b/libgo/go/net/http/fs.go index f61c138..c7a58a6 100644 --- a/libgo/go/net/http/fs.go +++ b/libgo/go/net/http/fs.go @@ -34,7 +34,7 @@ import ( type Dir string func (d Dir) Open(name string) (File, error) { - if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) || strings.Contains(name, "\x00") { return nil, errors.New("http: invalid character in file path") } @@ -96,7 +96,7 @@ func dirList(w ResponseWriter, f File) { } // ServeContent replies to the request using the content in the -// provided ReadSeeker. The main benefit of ServeContent over io.Copy +// provided ReadSeeker. The main benefit of ServeContent over io.Copy // is that it handles Range requests properly, sets the MIME type, and // handles If-Modified-Since requests. // @@ -108,7 +108,7 @@ func dirList(w ResponseWriter, f File) { // never sent in the response. // // If modtime is not the zero time or Unix epoch, ServeContent -// includes it in a Last-Modified header in the response. If the +// includes it in a Last-Modified header in the response. If the // request includes an If-Modified-Since header, ServeContent uses // modtime to decide whether the content needs to be sent at all. // @@ -121,11 +121,11 @@ func dirList(w ResponseWriter, f File) { // Note that *os.File implements the io.ReadSeeker interface. func ServeContent(w ResponseWriter, req *Request, name string, modtime time.Time, content io.ReadSeeker) { sizeFunc := func() (int64, error) { - size, err := content.Seek(0, os.SEEK_END) + size, err := content.Seek(0, io.SeekEnd) if err != nil { return 0, errSeeker } - _, err = content.Seek(0, os.SEEK_SET) + _, err = content.Seek(0, io.SeekStart) if err != nil { return 0, errSeeker } @@ -166,7 +166,7 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, var buf [sniffLen]byte n, _ := io.ReadFull(content, buf[:]) ctype = DetectContentType(buf[:n]) - _, err := content.Seek(0, os.SEEK_SET) // rewind to output whole file + _, err := content.Seek(0, io.SeekStart) // rewind to output whole file if err != nil { Error(w, "seeker can't seek", StatusInternalServerError) return @@ -196,7 +196,7 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, // The total number of bytes in all the ranges // is larger than the size of the file by // itself, so this is probably an attack, or a - // dumb client. Ignore the range request. + // dumb client. Ignore the range request. ranges = nil } switch { @@ -213,7 +213,7 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, // A response to a request for a single range MUST NOT // be sent using the multipart/byteranges media type." ra := ranges[0] - if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil { + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) return } @@ -236,7 +236,7 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, pw.CloseWithError(err) return } - if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil { + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { pw.CloseWithError(err) return } @@ -291,7 +291,7 @@ func checkLastModified(w ResponseWriter, r *Request, modtime time.Time) bool { // checkETag implements If-None-Match and If-Range checks. // // The ETag or modtime must have been previously set in the -// ResponseWriter's headers. The modtime is only compared at second +// ResponseWriter's headers. The modtime is only compared at second // granularity and may be the zero value to mean unknown. // // The return value is the effective request "Range" header to use and @@ -336,7 +336,7 @@ func checkETag(w ResponseWriter, r *Request, modtime time.Time) (rangeReq string } // TODO(bradfitz): deal with comma-separated or multiple-valued - // list of If-None-match values. For now just handle the common + // list of If-None-match values. For now just handle the common // case of a single item. if inm == etag || inm == "*" { h := w.Header() @@ -393,6 +393,15 @@ func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirec } } + // redirect if the directory name doesn't end in a slash + if d.IsDir() { + url := r.URL.Path + if url[len(url)-1] != '/' { + localRedirect(w, r, path.Base(url)+"/") + return + } + } + // use contents of index.html for directory, if present if d.IsDir() { index := strings.TrimSuffix(name, "/") + indexPage @@ -451,7 +460,7 @@ func localRedirect(w ResponseWriter, r *Request, newPath string) { // ServeFile replies to the request with the contents of the named // file or directory. // -// If the provided file or direcory name is a relative path, it is +// If the provided file or directory name is a relative path, it is // interpreted relative to the current directory and may ascend to parent // directories. If the provided name is constructed from user input, it // should be sanitized before calling ServeFile. As a precaution, ServeFile diff --git a/libgo/go/net/http/fs_test.go b/libgo/go/net/http/fs_test.go index cf5b63c..22be389 100644 --- a/libgo/go/net/http/fs_test.go +++ b/libgo/go/net/http/fs_test.go @@ -24,7 +24,6 @@ import ( "reflect" "regexp" "runtime" - "strconv" "strings" "testing" "time" @@ -39,8 +38,6 @@ type wantRange struct { start, end int64 // range [start,end) } -var itoa = strconv.Itoa - var ServeFileRangeTests = []struct { r string code int @@ -508,6 +505,24 @@ func TestServeFileFromCWD(t *testing.T) { } } +// Issue 13996 +func TestServeDirWithoutTrailingSlash(t *testing.T) { + e := "/testdata/" + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ServeFile(w, r, ".") + })) + defer ts.Close() + r, err := Get(ts.URL + "/testdata") + if err != nil { + t.Fatal(err) + } + r.Body.Close() + if g := r.Request.URL.Path; g != e { + t.Errorf("got %s, want %s", g, e) + } +} + // Tests that ServeFile doesn't add a Content-Length if a Content-Encoding is // specified. func TestServeFileWithContentEncoding_h1(t *testing.T) { testServeFileWithContentEncoding(t, h1Mode) } @@ -963,9 +978,9 @@ func TestLinuxSendfile(t *testing.T) { syscalls := "sendfile,sendfile64" switch runtime.GOARCH { - case "mips64", "mips64le", "alpha": - // mips64 strace doesn't support sendfile64 and will error out - // if we specify that with `-e trace='. + case "mips64", "mips64le", "s390x", "alpha": + // strace on the above platforms doesn't support sendfile64 + // and will error out if we specify that with `-e trace='. syscalls = "sendfile" } diff --git a/libgo/go/net/http/h2_bundle.go b/libgo/go/net/http/h2_bundle.go index 4e19b3e..db77455 100644 --- a/libgo/go/net/http/h2_bundle.go +++ b/libgo/go/net/http/h2_bundle.go @@ -1,5 +1,5 @@ // Code generated by golang.org/x/tools/cmd/bundle. -//go:generate bundle -o h2_bundle.go -prefix http2 -import golang.org/x/net/http2/hpack=internal/golang.org/x/net/http2/hpack golang.org/x/net/http2 +//go:generate bundle -o h2_bundle.go -prefix http2 golang.org/x/net/http2 // Package http2 implements the HTTP/2 protocol. // @@ -20,15 +20,16 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "crypto/tls" "encoding/binary" "errors" "fmt" - "internal/golang.org/x/net/http2/hpack" "io" "io/ioutil" "log" "net" + "net/http/httptrace" "net/textproto" "net/url" "os" @@ -39,6 +40,9 @@ import ( "strings" "sync" "time" + + "golang_org/x/net/http2/hpack" + "golang_org/x/net/lex/httplex" ) // ClientConnPool manages a pool of HTTP/2 client connections. @@ -47,6 +51,18 @@ type http2ClientConnPool interface { MarkDead(*http2ClientConn) } +// clientConnPoolIdleCloser is the interface implemented by ClientConnPool +// implementations which can close their idle connections. +type http2clientConnPoolIdleCloser interface { + http2ClientConnPool + closeIdleConnections() +} + +var ( + _ http2clientConnPoolIdleCloser = (*http2clientConnPool)(nil) + _ http2clientConnPoolIdleCloser = http2noDialClientConnPool{} +) + // TODO: use singleflight for dialing and addConnCalls? type http2clientConnPool struct { t *http2Transport @@ -247,6 +263,15 @@ func http2filterOutClientConn(in []*http2ClientConn, exclude *http2ClientConn) [ return out } +// noDialClientConnPool is an implementation of http2.ClientConnPool +// which never dials. We let the HTTP/1.1 client dial and use its TLS +// connection instead. +type http2noDialClientConnPool struct{ *http2clientConnPool } + +func (p http2noDialClientConnPool) GetClientConn(req *Request, addr string) (*http2ClientConn, error) { + return p.getClientConn(req, addr, http2noDialOnMiss) +} + func http2configureTransport(t1 *Transport) (*http2Transport, error) { connPool := new(http2clientConnPool) t2 := &http2Transport{ @@ -267,7 +292,7 @@ func http2configureTransport(t1 *Transport) (*http2Transport, error) { t1.TLSClientConfig.NextProtos = append(t1.TLSClientConfig.NextProtos, "http/1.1") } upgradeFn := func(authority string, c *tls.Conn) RoundTripper { - addr := http2authorityAddr(authority) + addr := http2authorityAddr("https", authority) if used, err := connPool.addConnIfNeeded(addr, t2, c); err != nil { go c.Close() return http2erringRoundTripper{err} @@ -299,15 +324,6 @@ func http2registerHTTPSProtocol(t *Transport, rt RoundTripper) (err error) { return nil } -// noDialClientConnPool is an implementation of http2.ClientConnPool -// which never dials. We let the HTTP/1.1 client dial and use its TLS -// connection instead. -type http2noDialClientConnPool struct{ *http2clientConnPool } - -func (p http2noDialClientConnPool) GetClientConn(req *Request, addr string) (*http2ClientConn, error) { - return p.getClientConn(req, addr, http2noDialOnMiss) -} - // noDialH2RoundTripper is a RoundTripper which only tries to complete the request // if there's already has a cached connection to the host. type http2noDialH2RoundTripper struct{ t *http2Transport } @@ -403,6 +419,35 @@ func (e http2connError) Error() string { return fmt.Sprintf("http2: connection error: %v: %v", e.Code, e.Reason) } +type http2pseudoHeaderError string + +func (e http2pseudoHeaderError) Error() string { + return fmt.Sprintf("invalid pseudo-header %q", string(e)) +} + +type http2duplicatePseudoHeaderError string + +func (e http2duplicatePseudoHeaderError) Error() string { + return fmt.Sprintf("duplicate pseudo-header %q", string(e)) +} + +type http2headerFieldNameError string + +func (e http2headerFieldNameError) Error() string { + return fmt.Sprintf("invalid header field name %q", string(e)) +} + +type http2headerFieldValueError string + +func (e http2headerFieldValueError) Error() string { + return fmt.Sprintf("invalid header field value %q", string(e)) +} + +var ( + http2errMixPseudoHeaderTypes = errors.New("mix of request and response pseudo headers") + http2errPseudoAfterRegular = errors.New("pseudo header field after regular") +) + // fixedBuffer is an io.ReadWriter backed by a fixed size buffer. // It never allocates, but moves old data as new data is written. type http2fixedBuffer struct { @@ -743,7 +788,7 @@ type http2Frame interface { type http2Framer struct { r io.Reader lastFrame http2Frame - errReason string + errDetail error // lastHeaderStream is non-zero if the last frame was an // unfinished HEADERS/CONTINUATION. @@ -775,14 +820,33 @@ type http2Framer struct { // to return non-compliant frames or frame orders. // This is for testing and permits using the Framer to test // other HTTP/2 implementations' conformance to the spec. + // It is not compatible with ReadMetaHeaders. AllowIllegalReads bool + // ReadMetaHeaders if non-nil causes ReadFrame to merge + // HEADERS and CONTINUATION frames together and return + // MetaHeadersFrame instead. + ReadMetaHeaders *hpack.Decoder + + // MaxHeaderListSize is the http2 MAX_HEADER_LIST_SIZE. + // It's used only if ReadMetaHeaders is set; 0 means a sane default + // (currently 16MB) + // If the limit is hit, MetaHeadersFrame.Truncated is set true. + MaxHeaderListSize uint32 + logReads bool debugFramer *http2Framer // only use for logging written writes debugFramerBuf *bytes.Buffer } +func (fr *http2Framer) maxHeaderListSize() uint32 { + if fr.MaxHeaderListSize == 0 { + return 16 << 20 + } + return fr.MaxHeaderListSize +} + func (f *http2Framer) startWrite(ftype http2FrameType, flags http2Flags, streamID uint32) { f.wbuf = append(f.wbuf[:0], @@ -879,6 +943,17 @@ func (fr *http2Framer) SetMaxReadFrameSize(v uint32) { fr.maxReadSize = v } +// ErrorDetail returns a more detailed error of the last error +// returned by Framer.ReadFrame. For instance, if ReadFrame +// returns a StreamError with code PROTOCOL_ERROR, ErrorDetail +// will say exactly what was invalid. ErrorDetail is not guaranteed +// to return a non-nil value and like the rest of the http2 package, +// its return value is not protected by an API compatibility promise. +// ErrorDetail is reset after the next call to ReadFrame. +func (fr *http2Framer) ErrorDetail() error { + return fr.errDetail +} + // ErrFrameTooLarge is returned from Framer.ReadFrame when the peer // sends a frame that is larger than declared with SetMaxReadFrameSize. var http2ErrFrameTooLarge = errors.New("http2: frame too large") @@ -897,9 +972,10 @@ func http2terminalReadFrameError(err error) bool { // // If the frame is larger than previously set with SetMaxReadFrameSize, the // returned error is ErrFrameTooLarge. Other errors may be of type -// ConnectionError, StreamError, or anything else from from the underlying +// ConnectionError, StreamError, or anything else from the underlying // reader. func (fr *http2Framer) ReadFrame() (http2Frame, error) { + fr.errDetail = nil if fr.lastFrame != nil { fr.lastFrame.invalidate() } @@ -927,6 +1003,9 @@ func (fr *http2Framer) ReadFrame() (http2Frame, error) { if fr.logReads { log.Printf("http2: Framer %p: read %v", fr, http2summarizeFrame(f)) } + if fh.Type == http2FrameHeaders && fr.ReadMetaHeaders != nil { + return fr.readMetaFrame(f.(*http2HeadersFrame)) + } return f, nil } @@ -935,7 +1014,7 @@ func (fr *http2Framer) ReadFrame() (http2Frame, error) { // to the peer before hanging up on them. This might help others debug // their implementations. func (fr *http2Framer) connError(code http2ErrCode, reason string) error { - fr.errReason = reason + fr.errDetail = errors.New(reason) return http2ConnectionError(code) } @@ -1023,7 +1102,14 @@ func http2parseDataFrame(fh http2FrameHeader, payload []byte) (http2Frame, error return f, nil } -var http2errStreamID = errors.New("invalid streamid") +var ( + http2errStreamID = errors.New("invalid stream ID") + http2errDepStreamID = errors.New("invalid dependent stream ID") +) + +func http2validStreamIDOrZero(streamID uint32) bool { + return streamID&(1<<31) == 0 +} func http2validStreamID(streamID uint32) bool { return streamID != 0 && streamID&(1<<31) == 0 @@ -1389,8 +1475,8 @@ func (f *http2Framer) WriteHeaders(p http2HeadersFrameParam) error { } if !p.Priority.IsZero() { v := p.Priority.StreamDep - if !http2validStreamID(v) && !f.AllowIllegalWrites { - return errors.New("invalid dependent stream id") + if !http2validStreamIDOrZero(v) && !f.AllowIllegalWrites { + return http2errDepStreamID } if p.Priority.Exclusive { v |= 1 << 31 @@ -1458,6 +1544,9 @@ func (f *http2Framer) WritePriority(streamID uint32, p http2PriorityParam) error if !http2validStreamID(streamID) && !f.AllowIllegalWrites { return http2errStreamID } + if !http2validStreamIDOrZero(p.StreamDep) { + return http2errDepStreamID + } f.startWrite(http2FramePriority, 0, streamID) v := p.StreamDep if p.Exclusive { @@ -1669,6 +1758,193 @@ type http2headersEnder interface { HeadersEnded() bool } +type http2headersOrContinuation interface { + http2headersEnder + HeaderBlockFragment() []byte +} + +// A MetaHeadersFrame is the representation of one HEADERS frame and +// zero or more contiguous CONTINUATION frames and the decoding of +// their HPACK-encoded contents. +// +// This type of frame does not appear on the wire and is only returned +// by the Framer when Framer.ReadMetaHeaders is set. +type http2MetaHeadersFrame struct { + *http2HeadersFrame + + // Fields are the fields contained in the HEADERS and + // CONTINUATION frames. The underlying slice is owned by the + // Framer and must not be retained after the next call to + // ReadFrame. + // + // Fields are guaranteed to be in the correct http2 order and + // not have unknown pseudo header fields or invalid header + // field names or values. Required pseudo header fields may be + // missing, however. Use the MetaHeadersFrame.Pseudo accessor + // method access pseudo headers. + Fields []hpack.HeaderField + + // Truncated is whether the max header list size limit was hit + // and Fields is incomplete. The hpack decoder state is still + // valid, however. + Truncated bool +} + +// PseudoValue returns the given pseudo header field's value. +// The provided pseudo field should not contain the leading colon. +func (mh *http2MetaHeadersFrame) PseudoValue(pseudo string) string { + for _, hf := range mh.Fields { + if !hf.IsPseudo() { + return "" + } + if hf.Name[1:] == pseudo { + return hf.Value + } + } + return "" +} + +// RegularFields returns the regular (non-pseudo) header fields of mh. +// The caller does not own the returned slice. +func (mh *http2MetaHeadersFrame) RegularFields() []hpack.HeaderField { + for i, hf := range mh.Fields { + if !hf.IsPseudo() { + return mh.Fields[i:] + } + } + return nil +} + +// PseudoFields returns the pseudo header fields of mh. +// The caller does not own the returned slice. +func (mh *http2MetaHeadersFrame) PseudoFields() []hpack.HeaderField { + for i, hf := range mh.Fields { + if !hf.IsPseudo() { + return mh.Fields[:i] + } + } + return mh.Fields +} + +func (mh *http2MetaHeadersFrame) checkPseudos() error { + var isRequest, isResponse bool + pf := mh.PseudoFields() + for i, hf := range pf { + switch hf.Name { + case ":method", ":path", ":scheme", ":authority": + isRequest = true + case ":status": + isResponse = true + default: + return http2pseudoHeaderError(hf.Name) + } + + for _, hf2 := range pf[:i] { + if hf.Name == hf2.Name { + return http2duplicatePseudoHeaderError(hf.Name) + } + } + } + if isRequest && isResponse { + return http2errMixPseudoHeaderTypes + } + return nil +} + +func (fr *http2Framer) maxHeaderStringLen() int { + v := fr.maxHeaderListSize() + if uint32(int(v)) == v { + return int(v) + } + + return 0 +} + +// readMetaFrame returns 0 or more CONTINUATION frames from fr and +// merge them into into the provided hf and returns a MetaHeadersFrame +// with the decoded hpack values. +func (fr *http2Framer) readMetaFrame(hf *http2HeadersFrame) (*http2MetaHeadersFrame, error) { + if fr.AllowIllegalReads { + return nil, errors.New("illegal use of AllowIllegalReads with ReadMetaHeaders") + } + mh := &http2MetaHeadersFrame{ + http2HeadersFrame: hf, + } + var remainSize = fr.maxHeaderListSize() + var sawRegular bool + + var invalid error // pseudo header field errors + hdec := fr.ReadMetaHeaders + hdec.SetEmitEnabled(true) + hdec.SetMaxStringLength(fr.maxHeaderStringLen()) + hdec.SetEmitFunc(func(hf hpack.HeaderField) { + if !httplex.ValidHeaderFieldValue(hf.Value) { + invalid = http2headerFieldValueError(hf.Value) + } + isPseudo := strings.HasPrefix(hf.Name, ":") + if isPseudo { + if sawRegular { + invalid = http2errPseudoAfterRegular + } + } else { + sawRegular = true + if !http2validWireHeaderFieldName(hf.Name) { + invalid = http2headerFieldNameError(hf.Name) + } + } + + if invalid != nil { + hdec.SetEmitEnabled(false) + return + } + + size := hf.Size() + if size > remainSize { + hdec.SetEmitEnabled(false) + mh.Truncated = true + return + } + remainSize -= size + + mh.Fields = append(mh.Fields, hf) + }) + + defer hdec.SetEmitFunc(func(hf hpack.HeaderField) {}) + + var hc http2headersOrContinuation = hf + for { + frag := hc.HeaderBlockFragment() + if _, err := hdec.Write(frag); err != nil { + return nil, http2ConnectionError(http2ErrCodeCompression) + } + + if hc.HeadersEnded() { + break + } + if f, err := fr.ReadFrame(); err != nil { + return nil, err + } else { + hc = f.(*http2ContinuationFrame) + } + } + + mh.http2HeadersFrame.headerFragBuf = nil + mh.http2HeadersFrame.invalidate() + + if err := hdec.Close(); err != nil { + return nil, http2ConnectionError(http2ErrCodeCompression) + } + if invalid != nil { + fr.errDetail = invalid + return nil, http2StreamError{mh.StreamID, http2ErrCodeProtocol} + } + if err := mh.checkPseudos(); err != nil { + fr.errDetail = err + return nil, http2StreamError{mh.StreamID, http2ErrCodeProtocol} + } + return mh, nil +} + func http2summarizeFrame(f http2Frame) string { var buf bytes.Buffer f.Header().writeDebug(&buf) @@ -1712,7 +1988,111 @@ func http2summarizeFrame(f http2Frame) string { return buf.String() } -func http2requestCancel(req *Request) <-chan struct{} { return req.Cancel } +func http2transportExpectContinueTimeout(t1 *Transport) time.Duration { + return t1.ExpectContinueTimeout +} + +// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. +func http2isBadCipher(cipher uint16) bool { + switch cipher { + case tls.TLS_RSA_WITH_RC4_128_SHA, + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, + tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: + + return true + default: + return false + } +} + +type http2contextContext interface { + context.Context +} + +func http2serverConnBaseContext(c net.Conn, opts *http2ServeConnOpts) (ctx http2contextContext, cancel func()) { + ctx, cancel = context.WithCancel(context.Background()) + ctx = context.WithValue(ctx, LocalAddrContextKey, c.LocalAddr()) + if hs := opts.baseConfig(); hs != nil { + ctx = context.WithValue(ctx, ServerContextKey, hs) + } + return +} + +func http2contextWithCancel(ctx http2contextContext) (_ http2contextContext, cancel func()) { + return context.WithCancel(ctx) +} + +func http2requestWithContext(req *Request, ctx http2contextContext) *Request { + return req.WithContext(ctx) +} + +type http2clientTrace httptrace.ClientTrace + +func http2reqContext(r *Request) context.Context { return r.Context() } + +func http2setResponseUncompressed(res *Response) { res.Uncompressed = true } + +func http2traceGotConn(req *Request, cc *http2ClientConn) { + trace := httptrace.ContextClientTrace(req.Context()) + if trace == nil || trace.GotConn == nil { + return + } + ci := httptrace.GotConnInfo{Conn: cc.tconn} + cc.mu.Lock() + ci.Reused = cc.nextStreamID > 1 + ci.WasIdle = len(cc.streams) == 0 && ci.Reused + if ci.WasIdle && !cc.lastActive.IsZero() { + ci.IdleTime = time.Now().Sub(cc.lastActive) + } + cc.mu.Unlock() + + trace.GotConn(ci) +} + +func http2traceWroteHeaders(trace *http2clientTrace) { + if trace != nil && trace.WroteHeaders != nil { + trace.WroteHeaders() + } +} + +func http2traceGot100Continue(trace *http2clientTrace) { + if trace != nil && trace.Got100Continue != nil { + trace.Got100Continue() + } +} + +func http2traceWait100Continue(trace *http2clientTrace) { + if trace != nil && trace.Wait100Continue != nil { + trace.Wait100Continue() + } +} + +func http2traceWroteRequest(trace *http2clientTrace, err error) { + if trace != nil && trace.WroteRequest != nil { + trace.WroteRequest(httptrace.WroteRequestInfo{Err: err}) + } +} + +func http2traceFirstResponseByte(trace *http2clientTrace) { + if trace != nil && trace.GotFirstResponseByte != nil { + trace.GotFirstResponseByte() + } +} + +func http2requestTrace(req *Request) *http2clientTrace { + trace := httptrace.ContextClientTrace(req.Context()) + return (*http2clientTrace)(trace) +} var http2DebugGoroutines = os.Getenv("DEBUG_HTTP2_GOROUTINES") == "1" @@ -2070,57 +2450,23 @@ var ( http2errInvalidHeaderFieldValue = errors.New("http2: invalid header field value") ) -// validHeaderFieldName reports whether v is a valid header field name (key). -// RFC 7230 says: -// header-field = field-name ":" OWS field-value OWS -// field-name = token -// tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / -// "^" / "_" / " +// validWireHeaderFieldName reports whether v is a valid header field +// name (key). See httplex.ValidHeaderName for the base rules. +// // Further, http2 says: // "Just as in HTTP/1.x, header field names are strings of ASCII // characters that are compared in a case-insensitive // fashion. However, header field names MUST be converted to // lowercase prior to their encoding in HTTP/2. " -func http2validHeaderFieldName(v string) bool { +func http2validWireHeaderFieldName(v string) bool { if len(v) == 0 { return false } for _, r := range v { - if int(r) >= len(http2isTokenTable) || ('A' <= r && r <= 'Z') { - return false - } - if !http2isTokenTable[byte(r)] { + if !httplex.IsTokenRune(r) { return false } - } - return true -} - -// validHeaderFieldValue reports whether v is a valid header field value. -// -// RFC 7230 says: -// field-value = *( field-content / obs-fold ) -// obj-fold = N/A to http2, and deprecated -// field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] -// field-vchar = VCHAR / obs-text -// obs-text = %x80-FF -// VCHAR = "any visible [USASCII] character" -// -// http2 further says: "Similarly, HTTP/2 allows header field values -// that are not valid. While most of the values that can be encoded -// will not alter header field parsing, carriage return (CR, ASCII -// 0xd), line feed (LF, ASCII 0xa), and the zero character (NUL, ASCII -// 0x0) might be exploited by an attacker if they are translated -// verbatim. Any request or response that contains a character not -// permitted in a header field value MUST be treated as malformed -// (Section 8.1.2.6). Valid characters are defined by the -// field-content ABNF rule in Section 3.2 of [RFC7230]." -// -// This function does not (yet?) properly handle the rejection of -// strings that begin or end with SP or HTAB. -func http2validHeaderFieldValue(v string) bool { - for i := 0; i < len(v); i++ { - if b := v[i]; b < ' ' && b != '\t' || b == 0x7f { + if 'A' <= r && r <= 'Z' { return false } } @@ -2225,7 +2571,7 @@ func http2mustUint31(v int32) uint32 { } // bodyAllowedForStatus reports whether a given response status code -// permits a body. See RFC2616, section 4.4. +// permits a body. See RFC 2616, section 4.4. func http2bodyAllowedForStatus(status int) bool { switch { case status >= 100 && status <= 199: @@ -2251,90 +2597,44 @@ func (e *http2httpError) Temporary() bool { return true } var http2errTimeout error = &http2httpError{msg: "http2: timeout awaiting response headers", timeout: true} -var http2isTokenTable = [127]bool{ - '!': true, - '#': true, - '$': true, - '%': true, - '&': true, - '\'': true, - '*': true, - '+': true, - '-': true, - '.': true, - '0': true, - '1': true, - '2': true, - '3': true, - '4': true, - '5': true, - '6': true, - '7': true, - '8': true, - '9': true, - 'A': true, - 'B': true, - 'C': true, - 'D': true, - 'E': true, - 'F': true, - 'G': true, - 'H': true, - 'I': true, - 'J': true, - 'K': true, - 'L': true, - 'M': true, - 'N': true, - 'O': true, - 'P': true, - 'Q': true, - 'R': true, - 'S': true, - 'T': true, - 'U': true, - 'W': true, - 'V': true, - 'X': true, - 'Y': true, - 'Z': true, - '^': true, - '_': true, - '`': true, - 'a': true, - 'b': true, - 'c': true, - 'd': true, - 'e': true, - 'f': true, - 'g': true, - 'h': true, - 'i': true, - 'j': true, - 'k': true, - 'l': true, - 'm': true, - 'n': true, - 'o': true, - 'p': true, - 'q': true, - 'r': true, - 's': true, - 't': true, - 'u': true, - 'v': true, - 'w': true, - 'x': true, - 'y': true, - 'z': true, - '|': true, - '~': true, -} - type http2connectionStater interface { ConnectionState() tls.ConnectionState } +var http2sorterPool = sync.Pool{New: func() interface{} { return new(http2sorter) }} + +type http2sorter struct { + v []string // owned by sorter +} + +func (s *http2sorter) Len() int { return len(s.v) } + +func (s *http2sorter) Swap(i, j int) { s.v[i], s.v[j] = s.v[j], s.v[i] } + +func (s *http2sorter) Less(i, j int) bool { return s.v[i] < s.v[j] } + +// Keys returns the sorted keys of h. +// +// The returned slice is only valid until s used again or returned to +// its pool. +func (s *http2sorter) Keys(h Header) []string { + keys := s.v[:0] + for k := range h { + keys = append(keys, k) + } + s.v = keys + sort.Sort(s) + return keys +} + +func (s *http2sorter) SortStrings(ss []string) { + + save := s.v + s.v = ss + sort.Sort(s) + s.v = save +} + // pipe is a goroutine-safe io.Reader/io.Writer pair. It's like // io.Pipe except there are no PipeReader/PipeWriter halves, and the // underlying buffer is an interface. (io.Pipe is always unbuffered) @@ -2354,6 +2654,12 @@ type http2pipeBuffer interface { io.Reader } +func (p *http2pipe) Len() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.b.Len() +} + // Read waits until data is available and copies bytes // from the buffer into p. func (p *http2pipe) Read(d []byte) (n int, err error) { @@ -2653,10 +2959,14 @@ func (o *http2ServeConnOpts) handler() Handler { // // The opts parameter is optional. If nil, default values are used. func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { + baseCtx, cancel := http2serverConnBaseContext(c, opts) + defer cancel() + sc := &http2serverConn{ srv: s, hs: opts.baseConfig(), conn: c, + baseCtx: baseCtx, remoteAddrStr: c.RemoteAddr().String(), bw: http2newBufferedWriter(c), handler: opts.handler(), @@ -2675,13 +2985,14 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { serveG: http2newGoroutineLock(), pushEnabled: true, } + sc.flow.add(http2initialWindowSize) sc.inflow.add(http2initialWindowSize) sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) - sc.hpackDecoder = hpack.NewDecoder(http2initialHeaderTableSize, nil) - sc.hpackDecoder.SetMaxStringLength(sc.maxHeaderStringLen()) fr := http2NewFramer(sc.bw, c) + fr.ReadMetaHeaders = hpack.NewDecoder(http2initialHeaderTableSize, nil) + fr.MaxHeaderListSize = sc.maxHeaderListSize() fr.SetMaxReadFrameSize(s.maxReadFrameSize()) sc.framer = fr @@ -2711,27 +3022,6 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { sc.serve() } -// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. -func http2isBadCipher(cipher uint16) bool { - switch cipher { - case tls.TLS_RSA_WITH_RC4_128_SHA, - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: - - return true - default: - return false - } -} - func (sc *http2serverConn) rejectConn(err http2ErrCode, debug string) { sc.vlogf("http2: server rejecting conn: %v, %s", err, debug) @@ -2747,8 +3037,8 @@ type http2serverConn struct { conn net.Conn bw *http2bufferedWriter // writing to conn handler Handler + baseCtx http2contextContext framer *http2Framer - hpackDecoder *hpack.Decoder doneServing chan struct{} // closed when serverConn.serve ends readFrameCh chan http2readFrameResult // written by serverConn.readFrames wantWriteFrameCh chan http2frameWriteMsg // from handlers -> serve @@ -2775,7 +3065,6 @@ type http2serverConn struct { headerTableSize uint32 peerMaxHeaderListSize uint32 // zero means unknown (default) canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case - req http2requestParam // non-zero while reading request headers writingFrame bool // started write goroutine but haven't heard back on wroteFrameCh needsFrameFlush bool // last frame write wasn't a flush writeSched http2writeScheduler @@ -2784,21 +3073,13 @@ type http2serverConn struct { goAwayCode http2ErrCode shutdownTimerCh <-chan time.Time // nil until used shutdownTimer *time.Timer // nil until used + freeRequestBodyBuf []byte // if non-nil, a free initialWindowSize buffer for getRequestBodyBuf // Owned by the writeFrameAsync goroutine: headerWriteBuf bytes.Buffer hpackEncoder *hpack.Encoder } -func (sc *http2serverConn) maxHeaderStringLen() int { - v := sc.maxHeaderListSize() - if uint32(int(v)) == v { - return int(v) - } - - return 0 -} - func (sc *http2serverConn) maxHeaderListSize() uint32 { n := sc.hs.MaxHeaderBytes if n <= 0 { @@ -2811,21 +3092,6 @@ func (sc *http2serverConn) maxHeaderListSize() uint32 { return uint32(n + typicalHeaders*perFieldOverhead) } -// requestParam is the state of the next request, initialized over -// potentially several frames HEADERS + zero or more CONTINUATION -// frames. -type http2requestParam struct { - // stream is non-nil if we're reading (HEADER or CONTINUATION) - // frames for a request (but not DATA). - stream *http2stream - header Header - method, path string - scheme, authority string - sawRegularHeader bool // saw a non-pseudo header already - invalidHeader bool // an invalid header was seen - headerListSize int64 // actually uint32, but easier math this way -} - // stream represents a stream. This is the minimal metadata needed by // the serve goroutine. Most of the actual stream state is owned by // the http.Handler's goroutine in the responseWriter. Because the @@ -2835,10 +3101,12 @@ type http2requestParam struct { // responseWriter's state field. type http2stream struct { // immutable: - sc *http2serverConn - id uint32 - body *http2pipe // non-nil if expecting DATA frames - cw http2closeWaiter // closed wait stream transitions to closed state + sc *http2serverConn + id uint32 + body *http2pipe // non-nil if expecting DATA frames + cw http2closeWaiter // closed wait stream transitions to closed state + ctx http2contextContext + cancelCtx func() // owned by serverConn's serve loop: bodyBytes int64 // body bytes seen so far @@ -2852,6 +3120,8 @@ type http2stream struct { sentReset bool // only true once detached from streams map gotReset bool // only true once detacted from streams map gotTrailerHeader bool // HEADER frame for trailers was seen + wroteHeaders bool // whether we wrote headers (not status 100) + reqBuf []byte trailer Header // accumulated trailers reqTrailer Header // handler's Request.Trailer @@ -2952,83 +3222,6 @@ func (sc *http2serverConn) condlogf(err error, format string, args ...interface{ } } -func (sc *http2serverConn) onNewHeaderField(f hpack.HeaderField) { - sc.serveG.check() - if http2VerboseLogs { - sc.vlogf("http2: server decoded %v", f) - } - switch { - case !http2validHeaderFieldValue(f.Value): - sc.req.invalidHeader = true - case strings.HasPrefix(f.Name, ":"): - if sc.req.sawRegularHeader { - sc.logf("pseudo-header after regular header") - sc.req.invalidHeader = true - return - } - var dst *string - switch f.Name { - case ":method": - dst = &sc.req.method - case ":path": - dst = &sc.req.path - case ":scheme": - dst = &sc.req.scheme - case ":authority": - dst = &sc.req.authority - default: - - sc.logf("invalid pseudo-header %q", f.Name) - sc.req.invalidHeader = true - return - } - if *dst != "" { - sc.logf("duplicate pseudo-header %q sent", f.Name) - sc.req.invalidHeader = true - return - } - *dst = f.Value - case !http2validHeaderFieldName(f.Name): - sc.req.invalidHeader = true - default: - sc.req.sawRegularHeader = true - sc.req.header.Add(sc.canonicalHeader(f.Name), f.Value) - const headerFieldOverhead = 32 // per spec - sc.req.headerListSize += int64(len(f.Name)) + int64(len(f.Value)) + headerFieldOverhead - if sc.req.headerListSize > int64(sc.maxHeaderListSize()) { - sc.hpackDecoder.SetEmitEnabled(false) - } - } -} - -func (st *http2stream) onNewTrailerField(f hpack.HeaderField) { - sc := st.sc - sc.serveG.check() - if http2VerboseLogs { - sc.vlogf("http2: server decoded trailer %v", f) - } - switch { - case strings.HasPrefix(f.Name, ":"): - sc.req.invalidHeader = true - return - case !http2validHeaderFieldName(f.Name) || !http2validHeaderFieldValue(f.Value): - sc.req.invalidHeader = true - return - default: - key := sc.canonicalHeader(f.Name) - if st.trailer != nil { - vv := append(st.trailer[key], f.Value) - st.trailer[key] = vv - - // arbitrary; TODO: read spec about header list size limits wrt trailers - const tooBig = 1000 - if len(vv) >= tooBig { - sc.hpackDecoder.SetEmitEnabled(false) - } - } - } -} - func (sc *http2serverConn) canonicalHeader(v string) string { sc.serveG.check() cv, ok := http2commonCanonHeader[v] @@ -3063,10 +3256,11 @@ type http2readFrameResult struct { // It's run on its own goroutine. func (sc *http2serverConn) readFrames() { gate := make(http2gate) + gateDone := gate.Done for { f, err := sc.framer.ReadFrame() select { - case sc.readFrameCh <- http2readFrameResult{f, err, gate.Done}: + case sc.readFrameCh <- http2readFrameResult{f, err, gateDone}: case <-sc.doneServing: return } @@ -3290,7 +3484,21 @@ func (sc *http2serverConn) writeFrameFromHandler(wm http2frameWriteMsg) error { // If you're not on the serve goroutine, use writeFrameFromHandler instead. func (sc *http2serverConn) writeFrame(wm http2frameWriteMsg) { sc.serveG.check() - sc.writeSched.add(wm) + + var ignoreWrite bool + + switch wm.write.(type) { + case *http2writeResHeaders: + wm.stream.wroteHeaders = true + case http2write100ContinueHeadersFrame: + if wm.stream.wroteHeaders { + ignoreWrite = true + } + } + + if !ignoreWrite { + sc.writeSched.add(wm) + } sc.scheduleFrameWrite() } @@ -3511,10 +3719,8 @@ func (sc *http2serverConn) processFrame(f http2Frame) error { switch f := f.(type) { case *http2SettingsFrame: return sc.processSettings(f) - case *http2HeadersFrame: + case *http2MetaHeadersFrame: return sc.processHeaders(f) - case *http2ContinuationFrame: - return sc.processContinuation(f) case *http2WindowUpdateFrame: return sc.processWindowUpdate(f) case *http2PingFrame: @@ -3579,6 +3785,7 @@ func (sc *http2serverConn) processResetStream(f *http2RSTStreamFrame) error { } if st != nil { st.gotReset = true + st.cancelCtx() sc.closeStream(st, http2StreamError{f.StreamID, f.ErrCode}) } return nil @@ -3600,6 +3807,10 @@ func (sc *http2serverConn) closeStream(st *http2stream, err error) { } st.cw.Close() sc.writeSched.forgetStream(st.id) + if st.reqBuf != nil { + + sc.freeRequestBodyBuf = st.reqBuf + } } func (sc *http2serverConn) processSettings(f *http2SettingsFrame) error { @@ -3732,7 +3943,7 @@ func (st *http2stream) copyTrailersToHandlerRequest() { } } -func (sc *http2serverConn) processHeaders(f *http2HeadersFrame) error { +func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { sc.serveG.check() id := f.Header().StreamID if sc.inGoAway { @@ -3749,17 +3960,18 @@ func (sc *http2serverConn) processHeaders(f *http2HeadersFrame) error { return st.processTrailerHeaders(f) } - if id <= sc.maxStreamID || sc.req.stream != nil { + if id <= sc.maxStreamID { return http2ConnectionError(http2ErrCodeProtocol) } + sc.maxStreamID = id - if id > sc.maxStreamID { - sc.maxStreamID = id - } + ctx, cancelCtx := http2contextWithCancel(sc.baseCtx) st = &http2stream{ - sc: sc, - id: id, - state: http2stateOpen, + sc: sc, + id: id, + state: http2stateOpen, + ctx: ctx, + cancelCtx: cancelCtx, } if f.StreamEnded() { st.state = http2stateHalfClosedRemote @@ -3779,50 +3991,6 @@ func (sc *http2serverConn) processHeaders(f *http2HeadersFrame) error { if sc.curOpenStreams == 1 { sc.setConnState(StateActive) } - sc.req = http2requestParam{ - stream: st, - header: make(Header), - } - sc.hpackDecoder.SetEmitFunc(sc.onNewHeaderField) - sc.hpackDecoder.SetEmitEnabled(true) - return sc.processHeaderBlockFragment(st, f.HeaderBlockFragment(), f.HeadersEnded()) -} - -func (st *http2stream) processTrailerHeaders(f *http2HeadersFrame) error { - sc := st.sc - sc.serveG.check() - if st.gotTrailerHeader { - return http2ConnectionError(http2ErrCodeProtocol) - } - st.gotTrailerHeader = true - if !f.StreamEnded() { - return http2StreamError{st.id, http2ErrCodeProtocol} - } - sc.resetPendingRequest() - return st.processTrailerHeaderBlockFragment(f.HeaderBlockFragment(), f.HeadersEnded()) -} - -func (sc *http2serverConn) processContinuation(f *http2ContinuationFrame) error { - sc.serveG.check() - st := sc.streams[f.Header().StreamID] - if st.gotTrailerHeader { - return st.processTrailerHeaderBlockFragment(f.HeaderBlockFragment(), f.HeadersEnded()) - } - return sc.processHeaderBlockFragment(st, f.HeaderBlockFragment(), f.HeadersEnded()) -} - -func (sc *http2serverConn) processHeaderBlockFragment(st *http2stream, frag []byte, end bool) error { - sc.serveG.check() - if _, err := sc.hpackDecoder.Write(frag); err != nil { - return http2ConnectionError(http2ErrCodeCompression) - } - if !end { - return nil - } - if err := sc.hpackDecoder.Close(); err != nil { - return http2ConnectionError(http2ErrCodeCompression) - } - defer sc.resetPendingRequest() if sc.curOpenStreams > sc.advMaxStreams { if sc.unackedSettings == 0 { @@ -3833,7 +4001,7 @@ func (sc *http2serverConn) processHeaderBlockFragment(st *http2stream, frag []by return http2StreamError{st.id, http2ErrCodeRefusedStream} } - rw, req, err := sc.newWriterAndRequest() + rw, req, err := sc.newWriterAndRequest(st, f) if err != nil { return err } @@ -3845,36 +4013,42 @@ func (sc *http2serverConn) processHeaderBlockFragment(st *http2stream, frag []by st.declBodyBytes = req.ContentLength handler := sc.handler.ServeHTTP - if !sc.hpackDecoder.EmitEnabled() { + if f.Truncated { handler = http2handleHeaderListTooLong + } else if err := http2checkValidHTTP2Request(req); err != nil { + handler = http2new400Handler(err) } go sc.runHandler(rw, req, handler) return nil } -func (st *http2stream) processTrailerHeaderBlockFragment(frag []byte, end bool) error { +func (st *http2stream) processTrailerHeaders(f *http2MetaHeadersFrame) error { sc := st.sc sc.serveG.check() - sc.hpackDecoder.SetEmitFunc(st.onNewTrailerField) - if _, err := sc.hpackDecoder.Write(frag); err != nil { - return http2ConnectionError(http2ErrCodeCompression) + if st.gotTrailerHeader { + return http2ConnectionError(http2ErrCodeProtocol) } - if !end { - return nil + st.gotTrailerHeader = true + if !f.StreamEnded() { + return http2StreamError{st.id, http2ErrCodeProtocol} } - rp := &sc.req - if rp.invalidHeader { - return http2StreamError{rp.stream.id, http2ErrCodeProtocol} + if len(f.PseudoFields()) > 0 { + return http2StreamError{st.id, http2ErrCodeProtocol} } + if st.trailer != nil { + for _, hf := range f.RegularFields() { + key := sc.canonicalHeader(hf.Name) + if !http2ValidTrailerHeader(key) { - err := sc.hpackDecoder.Close() - st.endStream() - if err != nil { - return http2ConnectionError(http2ErrCodeCompression) + return http2StreamError{st.id, http2ErrCodeProtocol} + } + st.trailer[key] = append(st.trailer[key], hf.Value) + } } + st.endStream() return nil } @@ -3912,59 +4086,56 @@ func http2adjustStreamPriority(streams map[uint32]*http2stream, streamID uint32, } } -// resetPendingRequest zeros out all state related to a HEADERS frame -// and its zero or more CONTINUATION frames sent to start a new -// request. -func (sc *http2serverConn) resetPendingRequest() { +func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHeadersFrame) (*http2responseWriter, *Request, error) { sc.serveG.check() - sc.req = http2requestParam{} -} -func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request, error) { - sc.serveG.check() - rp := &sc.req - - if rp.invalidHeader { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} - } + method := f.PseudoValue("method") + path := f.PseudoValue("path") + scheme := f.PseudoValue("scheme") + authority := f.PseudoValue("authority") - isConnect := rp.method == "CONNECT" + isConnect := method == "CONNECT" if isConnect { - if rp.path != "" || rp.scheme != "" || rp.authority == "" { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} + if path != "" || scheme != "" || authority == "" { + return nil, nil, http2StreamError{f.StreamID, http2ErrCodeProtocol} } - } else if rp.method == "" || rp.path == "" || - (rp.scheme != "https" && rp.scheme != "http") { + } else if method == "" || path == "" || + (scheme != "https" && scheme != "http") { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} + return nil, nil, http2StreamError{f.StreamID, http2ErrCodeProtocol} } - bodyOpen := rp.stream.state == http2stateOpen - if rp.method == "HEAD" && bodyOpen { + bodyOpen := !f.StreamEnded() + if method == "HEAD" && bodyOpen { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} + return nil, nil, http2StreamError{f.StreamID, http2ErrCodeProtocol} } var tlsState *tls.ConnectionState // nil if not scheme https - if rp.scheme == "https" { + if scheme == "https" { tlsState = sc.tlsState } - authority := rp.authority + + header := make(Header) + for _, hf := range f.RegularFields() { + header.Add(sc.canonicalHeader(hf.Name), hf.Value) + } + if authority == "" { - authority = rp.header.Get("Host") + authority = header.Get("Host") } - needsContinue := rp.header.Get("Expect") == "100-continue" + needsContinue := header.Get("Expect") == "100-continue" if needsContinue { - rp.header.Del("Expect") + header.Del("Expect") } - if cookies := rp.header["Cookie"]; len(cookies) > 1 { - rp.header.Set("Cookie", strings.Join(cookies, "; ")) + if cookies := header["Cookie"]; len(cookies) > 1 { + header.Set("Cookie", strings.Join(cookies, "; ")) } // Setup Trailers var trailer Header - for _, v := range rp.header["Trailer"] { + for _, v := range header["Trailer"] { for _, key := range strings.Split(v, ",") { key = CanonicalHeaderKey(strings.TrimSpace(key)) switch key { @@ -3978,31 +4149,31 @@ func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request } } } - delete(rp.header, "Trailer") + delete(header, "Trailer") body := &http2requestBody{ conn: sc, - stream: rp.stream, + stream: st, needsContinue: needsContinue, } var url_ *url.URL var requestURI string if isConnect { - url_ = &url.URL{Host: rp.authority} - requestURI = rp.authority + url_ = &url.URL{Host: authority} + requestURI = authority } else { var err error - url_, err = url.ParseRequestURI(rp.path) + url_, err = url.ParseRequestURI(path) if err != nil { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} + return nil, nil, http2StreamError{f.StreamID, http2ErrCodeProtocol} } - requestURI = rp.path + requestURI = path } req := &Request{ - Method: rp.method, + Method: method, URL: url_, RemoteAddr: sc.remoteAddrStr, - Header: rp.header, + Header: header, RequestURI: requestURI, Proto: "HTTP/2.0", ProtoMajor: 2, @@ -4012,12 +4183,16 @@ func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request Body: body, Trailer: trailer, } + req = http2requestWithContext(req, st.ctx) if bodyOpen { + + buf := make([]byte, http2initialWindowSize) + body.pipe = &http2pipe{ - b: &http2fixedBuffer{buf: make([]byte, http2initialWindowSize)}, + b: &http2fixedBuffer{buf: buf}, } - if vv, ok := rp.header["Content-Length"]; ok { + if vv, ok := header["Content-Length"]; ok { req.ContentLength, _ = strconv.ParseInt(vv[0], 10, 64) } else { req.ContentLength = -1 @@ -4030,7 +4205,7 @@ func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request rws.conn = sc rws.bw = bwSave rws.bw.Reset(http2chunkWriter{rws}) - rws.stream = rp.stream + rws.stream = st rws.req = req rws.body = body @@ -4038,10 +4213,20 @@ func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request return rw, req, nil } +func (sc *http2serverConn) getRequestBodyBuf() []byte { + sc.serveG.check() + if buf := sc.freeRequestBodyBuf; buf != nil { + sc.freeRequestBodyBuf = nil + return buf + } + return make([]byte, http2initialWindowSize) +} + // Run on its own goroutine. func (sc *http2serverConn) runHandler(rw *http2responseWriter, req *Request, handler func(ResponseWriter, *Request)) { didPanic := true defer func() { + rw.rws.stream.cancelCtx() if didPanic { e := recover() // Same as net/http: @@ -4190,7 +4375,7 @@ type http2requestBody struct { func (b *http2requestBody) Close() error { if b.pipe != nil { - b.pipe.CloseWithError(http2errClosedBody) + b.pipe.BreakWithError(http2errClosedBody) } b.closed = true return nil @@ -4265,9 +4450,9 @@ func (rws *http2responseWriterState) hasTrailers() bool { return len(rws.trailer // written in the trailers at the end of the response. func (rws *http2responseWriterState) declareTrailer(k string) { k = CanonicalHeaderKey(k) - switch k { - case "Transfer-Encoding", "Content-Length", "Trailer": + if !http2ValidTrailerHeader(k) { + rws.conn.logf("ignoring invalid trailer %q", k) return } if !http2strSliceContains(rws.trailers, k) { @@ -4408,7 +4593,12 @@ func (rws *http2responseWriterState) promoteUndeclaredTrailers() { rws.declareTrailer(trailerKey) rws.handlerHeader[CanonicalHeaderKey(trailerKey)] = vv } - sort.Strings(rws.trailers) + + if len(rws.trailers) > 1 { + sorter := http2sorterPool.Get().(*http2sorter) + sorter.SortStrings(rws.trailers) + http2sorterPool.Put(sorter) + } } func (w *http2responseWriter) Flush() { @@ -4552,6 +4742,72 @@ func http2foreachHeaderElement(v string, fn func(string)) { } } +// From http://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.2 +var http2connHeaders = []string{ + "Connection", + "Keep-Alive", + "Proxy-Connection", + "Transfer-Encoding", + "Upgrade", +} + +// checkValidHTTP2Request checks whether req is a valid HTTP/2 request, +// per RFC 7540 Section 8.1.2.2. +// The returned error is reported to users. +func http2checkValidHTTP2Request(req *Request) error { + for _, h := range http2connHeaders { + if _, ok := req.Header[h]; ok { + return fmt.Errorf("request header %q is not valid in HTTP/2", h) + } + } + te := req.Header["Te"] + if len(te) > 0 && (len(te) > 1 || (te[0] != "trailers" && te[0] != "")) { + return errors.New(`request header "TE" may only be "trailers" in HTTP/2`) + } + return nil +} + +func http2new400Handler(err error) HandlerFunc { + return func(w ResponseWriter, r *Request) { + Error(w, err.Error(), StatusBadRequest) + } +} + +// ValidTrailerHeader reports whether name is a valid header field name to appear +// in trailers. +// See: http://tools.ietf.org/html/rfc7230#section-4.1.2 +func http2ValidTrailerHeader(name string) bool { + name = CanonicalHeaderKey(name) + if strings.HasPrefix(name, "If-") || http2badTrailer[name] { + return false + } + return true +} + +var http2badTrailer = map[string]bool{ + "Authorization": true, + "Cache-Control": true, + "Connection": true, + "Content-Encoding": true, + "Content-Length": true, + "Content-Range": true, + "Content-Type": true, + "Expect": true, + "Host": true, + "Keep-Alive": true, + "Max-Forwards": true, + "Pragma": true, + "Proxy-Authenticate": true, + "Proxy-Authorization": true, + "Proxy-Connection": true, + "Range": true, + "Realm": true, + "Te": true, + "Trailer": true, + "Transfer-Encoding": true, + "Www-Authenticate": true, +} + const ( // transportDefaultConnFlow is how many connection-level flow control // tokens we give the server at start-up, past the default 64k. @@ -4601,6 +4857,10 @@ type http2Transport struct { // uncompressed. DisableCompression bool + // AllowHTTP, if true, permits HTTP/2 requests using the insecure, + // plain-text "http" scheme. Note that this does not enable h2c support. + AllowHTTP bool + // MaxHeaderListSize is the http2 SETTINGS_MAX_HEADER_LIST_SIZE to // send in the initial settings frame. It is how many bytes // of response headers are allow. Unlike the http2 spec, zero here @@ -4673,11 +4933,14 @@ type http2ClientConn struct { inflow http2flow // peer's conn-level flow control closed bool goAway *http2GoAwayFrame // if non-nil, the GoAwayFrame we received + goAwayDebug string // goAway frame's debug data, retained as a string streams map[uint32]*http2clientStream // client-initiated nextStreamID uint32 bw *bufio.Writer br *bufio.Reader fr *http2Framer + lastActive time.Time + // Settings from peer: maxFrameSize uint32 maxConcurrentStreams uint32 @@ -4695,10 +4958,12 @@ type http2ClientConn struct { type http2clientStream struct { cc *http2ClientConn req *Request + trace *http2clientTrace // or nil ID uint32 resc chan http2resAndError bufPipe http2pipe // buffered pipe with the flow-controlled response payload requestedGzip bool + on100 func() // optional code to run if get a 100 continue response flow http2flow // guarded by cc.mu inflow http2flow // guarded by cc.mu @@ -4712,36 +4977,43 @@ type http2clientStream struct { done chan struct{} // closed when stream remove from cc.streams map; close calls guarded by cc.mu // owned by clientConnReadLoop: - pastHeaders bool // got HEADERS w/ END_HEADERS - pastTrailers bool // got second HEADERS frame w/ END_HEADERS + firstByte bool // got the first response byte + pastHeaders bool // got first MetaHeadersFrame (actual headers) + pastTrailers bool // got optional second MetaHeadersFrame (trailers) trailer Header // accumulated trailers resTrailer *Header // client's Response.Trailer } // awaitRequestCancel runs in its own goroutine and waits for the user -// to either cancel a RoundTrip request (using the provided -// Request.Cancel channel), or for the request to be done (any way it -// might be removed from the cc.streams map: peer reset, successful -// completion, TCP connection breakage, etc) -func (cs *http2clientStream) awaitRequestCancel(cancel <-chan struct{}) { - if cancel == nil { +// to cancel a RoundTrip request, its context to expire, or for the +// request to be done (any way it might be removed from the cc.streams +// map: peer reset, successful completion, TCP connection breakage, +// etc) +func (cs *http2clientStream) awaitRequestCancel(req *Request) { + ctx := http2reqContext(req) + if req.Cancel == nil && ctx.Done() == nil { return } select { - case <-cancel: + case <-req.Cancel: cs.bufPipe.CloseWithError(http2errRequestCanceled) cs.cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) + case <-ctx.Done(): + cs.bufPipe.CloseWithError(ctx.Err()) + cs.cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) case <-cs.done: } } -// checkReset reports any error sent in a RST_STREAM frame by the -// server. -func (cs *http2clientStream) checkReset() error { +// checkResetOrDone reports any error sent in a RST_STREAM frame by the +// server, or errStreamClosed if the stream is complete. +func (cs *http2clientStream) checkResetOrDone() error { select { case <-cs.peerReset: return cs.resetErr + case <-cs.done: + return http2errStreamClosed default: return nil } @@ -4789,26 +5061,31 @@ func (t *http2Transport) RoundTrip(req *Request) (*Response, error) { // authorityAddr returns a given authority (a host/IP, or host:port / ip:port) // and returns a host:port. The port 443 is added if needed. -func http2authorityAddr(authority string) (addr string) { +func http2authorityAddr(scheme string, authority string) (addr string) { if _, _, err := net.SplitHostPort(authority); err == nil { return authority } - return net.JoinHostPort(authority, "443") + port := "443" + if scheme == "http" { + port = "80" + } + return net.JoinHostPort(authority, port) } // RoundTripOpt is like RoundTrip, but takes options. func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Response, error) { - if req.URL.Scheme != "https" { + if !(req.URL.Scheme == "https" || (req.URL.Scheme == "http" && t.AllowHTTP)) { return nil, errors.New("http2: unsupported scheme") } - addr := http2authorityAddr(req.URL.Host) + addr := http2authorityAddr(req.URL.Scheme, req.URL.Host) for { cc, err := t.connPool().GetClientConn(req, addr) if err != nil { t.vlogf("http2: Transport failed to get client conn for %s: %v", addr, err) return nil, err } + http2traceGotConn(req, cc) res, err := cc.RoundTrip(req) if http2shouldRetryRequest(req, err) { continue @@ -4825,7 +5102,7 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res // connected from previous requests but are now sitting idle. // It does not interrupt any connections currently in use. func (t *http2Transport) CloseIdleConnections() { - if cp, ok := t.connPool().(*http2clientConnPool); ok { + if cp, ok := t.connPool().(http2clientConnPoolIdleCloser); ok { cp.closeIdleConnections() } } @@ -4857,8 +5134,12 @@ func (t *http2Transport) newTLSConfig(host string) *tls.Config { if t.TLSClientConfig != nil { *cfg = *t.TLSClientConfig } - cfg.NextProtos = []string{http2NextProtoTLS} - cfg.ServerName = host + if !http2strSliceContains(cfg.NextProtos, http2NextProtoTLS) { + cfg.NextProtos = append([]string{http2NextProtoTLS}, cfg.NextProtos...) + } + if cfg.ServerName == "" { + cfg.ServerName = host + } return cfg } @@ -4898,6 +5179,13 @@ func (t *http2Transport) disableKeepAlives() bool { return t.t1 != nil && t.t1.DisableKeepAlives } +func (t *http2Transport) expectContinueTimeout() time.Duration { + if t.t1 == nil { + return 0 + } + return http2transportExpectContinueTimeout(t.t1) +} + func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { if http2VerboseLogs { t.vlogf("http2: Transport creating client conn to %v", c.RemoteAddr()) @@ -4923,6 +5211,8 @@ func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { cc.bw = bufio.NewWriter(http2stickyErrWriter{c, &cc.werr}) cc.br = bufio.NewReader(c) cc.fr = http2NewFramer(cc.bw, cc.br) + cc.fr.ReadMetaHeaders = hpack.NewDecoder(http2initialHeaderTableSize, nil) + cc.fr.MaxHeaderListSize = t.maxHeaderListSize() cc.henc = hpack.NewEncoder(&cc.hbuf) @@ -4932,8 +5222,8 @@ func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { } initialSettings := []http2Setting{ - http2Setting{ID: http2SettingEnablePush, Val: 0}, - http2Setting{ID: http2SettingInitialWindowSize, Val: http2transportDefaultStreamFlow}, + {ID: http2SettingEnablePush, Val: 0}, + {ID: http2SettingInitialWindowSize, Val: http2transportDefaultStreamFlow}, } if max := t.maxHeaderListSize(); max != 0 { initialSettings = append(initialSettings, http2Setting{ID: http2SettingMaxHeaderListSize, Val: max}) @@ -4979,7 +5269,16 @@ func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { func (cc *http2ClientConn) setGoAway(f *http2GoAwayFrame) { cc.mu.Lock() defer cc.mu.Unlock() + + old := cc.goAway cc.goAway = f + + if cc.goAwayDebug == "" { + cc.goAwayDebug = string(f.DebugData()) + } + if old != nil && old.ErrCode != http2ErrCodeNo { + cc.goAway.ErrCode = old.ErrCode + } } func (cc *http2ClientConn) CanTakeNewRequest() bool { @@ -5093,6 +5392,30 @@ func http2checkConnHeaders(req *Request) error { return nil } +func http2bodyAndLength(req *Request) (body io.Reader, contentLen int64) { + body = req.Body + if body == nil { + return nil, 0 + } + if req.ContentLength != 0 { + return req.Body, req.ContentLength + } + + // We have a body but a zero content length. Test to see if + // it's actually zero or just unset. + var buf [1]byte + n, rerr := io.ReadFull(body, buf[:]) + if rerr != nil && rerr != io.EOF { + return http2errorReader{rerr}, -1 + } + if n == 1 { + + return io.MultiReader(bytes.NewReader(buf[:]), body), -1 + } + + return nil, 0 +} + func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { if err := http2checkConnHeaders(req); err != nil { return nil, err @@ -5104,67 +5427,62 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { } hasTrailers := trailers != "" - var body io.Reader = req.Body - contentLen := req.ContentLength - if req.Body != nil && contentLen == 0 { - // Test to see if it's actually zero or just unset. - var buf [1]byte - n, rerr := io.ReadFull(body, buf[:]) - if rerr != nil && rerr != io.EOF { - contentLen = -1 - body = http2errorReader{rerr} - } else if n == 1 { - - contentLen = -1 - body = io.MultiReader(bytes.NewReader(buf[:]), body) - } else { - - body = nil - } - } + body, contentLen := http2bodyAndLength(req) + hasBody := body != nil cc.mu.Lock() + cc.lastActive = time.Now() if cc.closed || !cc.canTakeNewRequestLocked() { cc.mu.Unlock() return nil, http2errClientConnUnusable } - cs := cc.newStream() - cs.req = req - hasBody := body != nil - + // TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere? + var requestedGzip bool if !cc.t.disableCompression() && req.Header.Get("Accept-Encoding") == "" && req.Header.Get("Range") == "" && req.Method != "HEAD" { - cs.requestedGzip = true + requestedGzip = true } - hdrs := cc.encodeHeaders(req, cs.requestedGzip, trailers, contentLen) + hdrs, err := cc.encodeHeaders(req, requestedGzip, trailers, contentLen) + if err != nil { + cc.mu.Unlock() + return nil, err + } + + cs := cc.newStream() + cs.req = req + cs.trace = http2requestTrace(req) + cs.requestedGzip = requestedGzip + bodyWriter := cc.t.getBodyWriterState(cs, body) + cs.on100 = bodyWriter.on100 + cc.wmu.Lock() endStream := !hasBody && !hasTrailers werr := cc.writeHeaders(cs.ID, endStream, hdrs) cc.wmu.Unlock() + http2traceWroteHeaders(cs.trace) cc.mu.Unlock() if werr != nil { if hasBody { req.Body.Close() + bodyWriter.cancel() } cc.forgetStreamID(cs.ID) + http2traceWroteRequest(cs.trace, werr) return nil, werr } var respHeaderTimer <-chan time.Time - var bodyCopyErrc chan error // result of body copy if hasBody { - bodyCopyErrc = make(chan error, 1) - go func() { - bodyCopyErrc <- cs.writeRequestBody(body, req.Body) - }() + bodyWriter.scheduleBodyWrite() } else { + http2traceWroteRequest(cs.trace, nil) if d := cc.responseHeaderTimeout(); d != 0 { timer := time.NewTimer(d) defer timer.Stop() @@ -5173,44 +5491,78 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { } readLoopResCh := cs.resc - requestCanceledCh := http2requestCancel(req) bodyWritten := false + ctx := http2reqContext(req) + + reFunc := func(re http2resAndError) (*Response, error) { + res := re.res + if re.err != nil || res.StatusCode > 299 { + bodyWriter.cancel() + cs.abortRequestBodyWrite(http2errStopReqBodyWrite) + } + if re.err != nil { + cc.forgetStreamID(cs.ID) + return nil, re.err + } + res.Request = req + res.TLS = cc.tlsState + return res, nil + } for { select { case re := <-readLoopResCh: - res := re.res - if re.err != nil || res.StatusCode > 299 { - - cs.abortRequestBodyWrite(http2errStopReqBodyWrite) - } - if re.err != nil { - cc.forgetStreamID(cs.ID) - return nil, re.err - } - res.Request = req - res.TLS = cc.tlsState - return res, nil + return reFunc(re) case <-respHeaderTimer: cc.forgetStreamID(cs.ID) if !hasBody || bodyWritten { cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) } else { + bodyWriter.cancel() cs.abortRequestBodyWrite(http2errStopReqBodyWriteAndCancel) } return nil, http2errTimeout - case <-requestCanceledCh: + case <-ctx.Done(): + select { + case re := <-readLoopResCh: + return reFunc(re) + default: + } + cc.forgetStreamID(cs.ID) + if !hasBody || bodyWritten { + cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) + } else { + bodyWriter.cancel() + cs.abortRequestBodyWrite(http2errStopReqBodyWriteAndCancel) + } + return nil, ctx.Err() + case <-req.Cancel: + select { + case re := <-readLoopResCh: + return reFunc(re) + default: + } cc.forgetStreamID(cs.ID) if !hasBody || bodyWritten { cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) } else { + bodyWriter.cancel() cs.abortRequestBodyWrite(http2errStopReqBodyWriteAndCancel) } return nil, http2errRequestCanceled case <-cs.peerReset: - + select { + case re := <-readLoopResCh: + return reFunc(re) + default: + } return nil, cs.resetErr - case err := <-bodyCopyErrc: + case err := <-bodyWriter.resc: + select { + case re := <-readLoopResCh: + return reFunc(re) + default: + } if err != nil { return nil, err } @@ -5268,6 +5620,7 @@ func (cs *http2clientStream) writeRequestBody(body io.Reader, bodyCloser io.Clos defer cc.putFrameScratchBuffer(buf) defer func() { + http2traceWroteRequest(cs.trace, err) cerr := bodyCloser.Close() if err == nil { @@ -5355,7 +5708,7 @@ func (cs *http2clientStream) awaitFlowControl(maxBytes int) (taken int32, err er if cs.stopReqBody != nil { return 0, cs.stopReqBody } - if err := cs.checkReset(); err != nil { + if err := cs.checkResetOrDone(); err != nil { return 0, err } if a := cs.flow.available(); a > 0 { @@ -5382,7 +5735,7 @@ type http2badStringError struct { func (e *http2badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) } // requires cc.mu be held. -func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trailers string, contentLength int64) []byte { +func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trailers string, contentLength int64) ([]byte, error) { cc.hbuf.Reset() host := req.Host @@ -5390,6 +5743,17 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail host = req.URL.Host } + for k, vv := range req.Header { + if !httplex.ValidHeaderFieldName(k) { + return nil, fmt.Errorf("invalid HTTP header name %q", k) + } + for _, v := range vv { + if !httplex.ValidHeaderFieldValue(v) { + return nil, fmt.Errorf("invalid HTTP header value %q for header %q", v, k) + } + } + } + cc.writeHeader(":authority", host) cc.writeHeader(":method", req.Method) if req.Method != "CONNECT" { @@ -5407,7 +5771,7 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail case "host", "content-length": continue - case "connection", "proxy-connection", "transfer-encoding", "upgrade": + case "connection", "proxy-connection", "transfer-encoding", "upgrade", "keep-alive": continue case "user-agent": @@ -5434,7 +5798,7 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail if !didUA { cc.writeHeader("user-agent", http2defaultUserAgent) } - return cc.hbuf.Bytes() + return cc.hbuf.Bytes(), nil } // shouldSendReqContentLength reports whether the http2.Transport should send @@ -5510,8 +5874,10 @@ func (cc *http2ClientConn) streamByID(id uint32, andRemove bool) *http2clientStr defer cc.mu.Unlock() cs := cc.streams[id] if andRemove && cs != nil && !cc.closed { + cc.lastActive = time.Now() delete(cc.streams, id) close(cs.done) + cc.cond.Broadcast() } return cs } @@ -5521,15 +5887,6 @@ type http2clientConnReadLoop struct { cc *http2ClientConn activeRes map[uint32]*http2clientStream // keyed by streamID closeWhenIdle bool - - hdec *hpack.Decoder - - // Fields reset on each HEADERS: - nextRes *Response - sawRegHeader bool // saw non-pseudo header - reqMalformed error // non-nil once known to be malformed - lastHeaderEndsStream bool - headerListSize int64 // actually uint32, but easier math this way } // readLoop runs in its own goroutine and reads and dispatches frames. @@ -5538,7 +5895,6 @@ func (cc *http2ClientConn) readLoop() { cc: cc, activeRes: make(map[uint32]*http2clientStream), } - rl.hdec = hpack.NewDecoder(http2initialHeaderTableSize, rl.onNewHeaderField) defer rl.cleanup() cc.readerErr = rl.run() @@ -5549,6 +5905,19 @@ func (cc *http2ClientConn) readLoop() { } } +// GoAwayError is returned by the Transport when the server closes the +// TCP connection after sending a GOAWAY frame. +type http2GoAwayError struct { + LastStreamID uint32 + ErrCode http2ErrCode + DebugData string +} + +func (e http2GoAwayError) Error() string { + return fmt.Sprintf("http2: server sent GOAWAY and closed the connection; LastStreamID=%v, ErrCode=%v, debug=%q", + e.LastStreamID, e.ErrCode, e.DebugData) +} + func (rl *http2clientConnReadLoop) cleanup() { cc := rl.cc defer cc.tconn.Close() @@ -5556,10 +5925,18 @@ func (rl *http2clientConnReadLoop) cleanup() { defer close(cc.readerDone) err := cc.readerErr + cc.mu.Lock() if err == io.EOF { - err = io.ErrUnexpectedEOF + if cc.goAway != nil { + err = http2GoAwayError{ + LastStreamID: cc.goAway.LastStreamID, + ErrCode: cc.goAway.ErrCode, + DebugData: cc.goAwayDebug, + } + } else { + err = io.ErrUnexpectedEOF + } } - cc.mu.Lock() for _, cs := range rl.activeRes { cs.bufPipe.CloseWithError(err) } @@ -5585,8 +5962,10 @@ func (rl *http2clientConnReadLoop) run() error { cc.vlogf("Transport readFrame error: (%T) %v", err, err) } if se, ok := err.(http2StreamError); ok { - - return se + if cs := cc.streamByID(se.StreamID, true); cs != nil { + rl.endStreamError(cs, cc.fr.errDetail) + } + continue } else if err != nil { return err } @@ -5596,13 +5975,10 @@ func (rl *http2clientConnReadLoop) run() error { maybeIdle := false switch f := f.(type) { - case *http2HeadersFrame: + case *http2MetaHeadersFrame: err = rl.processHeaders(f) maybeIdle = true gotReply = true - case *http2ContinuationFrame: - err = rl.processContinuation(f) - maybeIdle = true case *http2DataFrame: err = rl.processData(f) maybeIdle = true @@ -5632,83 +6008,105 @@ func (rl *http2clientConnReadLoop) run() error { } } -func (rl *http2clientConnReadLoop) processHeaders(f *http2HeadersFrame) error { - rl.sawRegHeader = false - rl.reqMalformed = nil - rl.lastHeaderEndsStream = f.StreamEnded() - rl.headerListSize = 0 - rl.nextRes = &Response{ - Proto: "HTTP/2.0", - ProtoMajor: 2, - Header: make(Header), - } - rl.hdec.SetEmitEnabled(true) - return rl.processHeaderBlockFragment(f.HeaderBlockFragment(), f.StreamID, f.HeadersEnded()) -} - -func (rl *http2clientConnReadLoop) processContinuation(f *http2ContinuationFrame) error { - return rl.processHeaderBlockFragment(f.HeaderBlockFragment(), f.StreamID, f.HeadersEnded()) -} - -func (rl *http2clientConnReadLoop) processHeaderBlockFragment(frag []byte, streamID uint32, finalFrag bool) error { +func (rl *http2clientConnReadLoop) processHeaders(f *http2MetaHeadersFrame) error { cc := rl.cc - streamEnded := rl.lastHeaderEndsStream - cs := cc.streamByID(streamID, streamEnded && finalFrag) + cs := cc.streamByID(f.StreamID, f.StreamEnded()) if cs == nil { return nil } - if cs.pastHeaders { - rl.hdec.SetEmitFunc(func(f hpack.HeaderField) { rl.onNewTrailerField(cs, f) }) - } else { - rl.hdec.SetEmitFunc(rl.onNewHeaderField) - } - _, err := rl.hdec.Write(frag) - if err != nil { - return http2ConnectionError(http2ErrCodeCompression) - } - if finalFrag { - if err := rl.hdec.Close(); err != nil { - return http2ConnectionError(http2ErrCodeCompression) - } - } + if !cs.firstByte { + if cs.trace != nil { - if !finalFrag { - return nil + http2traceFirstResponseByte(cs.trace) + } + cs.firstByte = true } - if !cs.pastHeaders { cs.pastHeaders = true } else { + return rl.processTrailers(cs, f) + } - if cs.pastTrailers { - - return http2ConnectionError(http2ErrCodeProtocol) + res, err := rl.handleResponse(cs, f) + if err != nil { + if _, ok := err.(http2ConnectionError); ok { + return err } - cs.pastTrailers = true - if !streamEnded { - return http2ConnectionError(http2ErrCodeProtocol) - } - rl.endStream(cs) + cs.cc.writeStreamReset(f.StreamID, http2ErrCodeProtocol, err) + cs.resc <- http2resAndError{err: err} return nil } + if res == nil { - if rl.reqMalformed != nil { - cs.resc <- http2resAndError{err: rl.reqMalformed} - rl.cc.writeStreamReset(cs.ID, http2ErrCodeProtocol, rl.reqMalformed) return nil } + if res.Body != http2noBody { + rl.activeRes[cs.ID] = cs + } + cs.resTrailer = &res.Trailer + cs.resc <- http2resAndError{res: res} + return nil +} - res := rl.nextRes +// may return error types nil, or ConnectionError. Any other error value +// is a StreamError of type ErrCodeProtocol. The returned error in that case +// is the detail. +// +// As a special case, handleResponse may return (nil, nil) to skip the +// frame (currently only used for 100 expect continue). This special +// case is going away after Issue 13851 is fixed. +func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http2MetaHeadersFrame) (*Response, error) { + if f.Truncated { + return nil, http2errResponseHeaderListSize + } - if res.StatusCode == 100 { + status := f.PseudoValue("status") + if status == "" { + return nil, errors.New("missing status pseudo header") + } + statusCode, err := strconv.Atoi(status) + if err != nil { + return nil, errors.New("malformed non-numeric status pseudo header") + } + if statusCode == 100 { + http2traceGot100Continue(cs.trace) + if cs.on100 != nil { + cs.on100() + } cs.pastHeaders = false - return nil + return nil, nil + } + + header := make(Header) + res := &Response{ + Proto: "HTTP/2.0", + ProtoMajor: 2, + Header: header, + StatusCode: statusCode, + Status: status + " " + StatusText(statusCode), + } + for _, hf := range f.RegularFields() { + key := CanonicalHeaderKey(hf.Name) + if key == "Trailer" { + t := res.Trailer + if t == nil { + t = make(Header) + res.Trailer = t + } + http2foreachHeaderElement(hf.Value, func(v string) { + t[CanonicalHeaderKey(v)] = nil + }) + } else { + header[key] = append(header[key], hf.Value) + } } - if !streamEnded || cs.req.Method == "HEAD" { + streamEnded := f.StreamEnded() + isHead := cs.req.Method == "HEAD" + if !streamEnded || isHead { res.ContentLength = -1 if clens := res.Header["Content-Length"]; len(clens) == 1 { if clen64, err := strconv.ParseInt(clens[0], 10, 64); err == nil { @@ -5721,27 +6119,50 @@ func (rl *http2clientConnReadLoop) processHeaderBlockFragment(frag []byte, strea } } - if streamEnded { + if streamEnded || isHead { res.Body = http2noBody - } else { - buf := new(bytes.Buffer) - cs.bufPipe = http2pipe{b: buf} - cs.bytesRemain = res.ContentLength - res.Body = http2transportResponseBody{cs} - go cs.awaitRequestCancel(http2requestCancel(cs.req)) - - if cs.requestedGzip && res.Header.Get("Content-Encoding") == "gzip" { - res.Header.Del("Content-Encoding") - res.Header.Del("Content-Length") - res.ContentLength = -1 - res.Body = &http2gzipReader{body: res.Body} - } - rl.activeRes[cs.ID] = cs + return res, nil } - cs.resTrailer = &res.Trailer - cs.resc <- http2resAndError{res: res} - rl.nextRes = nil + buf := new(bytes.Buffer) + cs.bufPipe = http2pipe{b: buf} + cs.bytesRemain = res.ContentLength + res.Body = http2transportResponseBody{cs} + go cs.awaitRequestCancel(cs.req) + + if cs.requestedGzip && res.Header.Get("Content-Encoding") == "gzip" { + res.Header.Del("Content-Encoding") + res.Header.Del("Content-Length") + res.ContentLength = -1 + res.Body = &http2gzipReader{body: res.Body} + http2setResponseUncompressed(res) + } + return res, nil +} + +func (rl *http2clientConnReadLoop) processTrailers(cs *http2clientStream, f *http2MetaHeadersFrame) error { + if cs.pastTrailers { + + return http2ConnectionError(http2ErrCodeProtocol) + } + cs.pastTrailers = true + if !f.StreamEnded() { + + return http2ConnectionError(http2ErrCodeProtocol) + } + if len(f.PseudoFields()) > 0 { + + return http2ConnectionError(http2ErrCodeProtocol) + } + + trailer := make(Header) + for _, hf := range f.RegularFields() { + key := CanonicalHeaderKey(hf.Name) + trailer[key] = append(trailer[key], hf.Value) + } + cs.trailer = trailer + + rl.endStream(cs) return nil } @@ -5792,8 +6213,10 @@ func (b http2transportResponseBody) Read(p []byte) (n int, err error) { cc.inflow.add(connAdd) } if err == nil { - if v := cs.inflow.available(); v < http2transportDefaultStreamFlow-http2transportDefaultStreamMinRefresh { - streamAdd = http2transportDefaultStreamFlow - v + + v := int(cs.inflow.available()) + cs.bufPipe.Len() + if v < http2transportDefaultStreamFlow-http2transportDefaultStreamMinRefresh { + streamAdd = int32(http2transportDefaultStreamFlow - v) cs.inflow.add(streamAdd) } } @@ -5855,6 +6278,7 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { cc.mu.Unlock() if _, err := cs.bufPipe.Write(data); err != nil { + rl.endStreamError(cs, err) return err } } @@ -5869,11 +6293,14 @@ var http2errInvalidTrailers = errors.New("http2: invalid trailers") func (rl *http2clientConnReadLoop) endStream(cs *http2clientStream) { - err := io.EOF - code := cs.copyTrailers - if rl.reqMalformed != nil { - err = rl.reqMalformed - code = nil + rl.endStreamError(cs, nil) +} + +func (rl *http2clientConnReadLoop) endStreamError(cs *http2clientStream, err error) { + var code func() + if err == nil { + err = io.EOF + code = cs.copyTrailers } cs.bufPipe.closeWithErrorAndCode(err, code) delete(rl.activeRes, cs.ID) @@ -5997,113 +6424,6 @@ var ( http2errPseudoTrailers = errors.New("http2: invalid pseudo header in trailers") ) -func (rl *http2clientConnReadLoop) checkHeaderField(f hpack.HeaderField) bool { - if rl.reqMalformed != nil { - return false - } - - const headerFieldOverhead = 32 // per spec - rl.headerListSize += int64(len(f.Name)) + int64(len(f.Value)) + headerFieldOverhead - if max := rl.cc.t.maxHeaderListSize(); max != 0 && rl.headerListSize > int64(max) { - rl.hdec.SetEmitEnabled(false) - rl.reqMalformed = http2errResponseHeaderListSize - return false - } - - if !http2validHeaderFieldValue(f.Value) { - rl.reqMalformed = http2errInvalidHeaderFieldValue - return false - } - - isPseudo := strings.HasPrefix(f.Name, ":") - if isPseudo { - if rl.sawRegHeader { - rl.reqMalformed = errors.New("http2: invalid pseudo header after regular header") - return false - } - } else { - if !http2validHeaderFieldName(f.Name) { - rl.reqMalformed = http2errInvalidHeaderFieldName - return false - } - rl.sawRegHeader = true - } - - return true -} - -// onNewHeaderField runs on the readLoop goroutine whenever a new -// hpack header field is decoded. -func (rl *http2clientConnReadLoop) onNewHeaderField(f hpack.HeaderField) { - cc := rl.cc - if http2VerboseLogs { - cc.logf("http2: Transport decoded %v", f) - } - - if !rl.checkHeaderField(f) { - return - } - - isPseudo := strings.HasPrefix(f.Name, ":") - if isPseudo { - switch f.Name { - case ":status": - code, err := strconv.Atoi(f.Value) - if err != nil { - rl.reqMalformed = errors.New("http2: invalid :status") - return - } - rl.nextRes.Status = f.Value + " " + StatusText(code) - rl.nextRes.StatusCode = code - default: - - rl.reqMalformed = fmt.Errorf("http2: unknown response pseudo header %q", f.Name) - } - return - } - - key := CanonicalHeaderKey(f.Name) - if key == "Trailer" { - t := rl.nextRes.Trailer - if t == nil { - t = make(Header) - rl.nextRes.Trailer = t - } - http2foreachHeaderElement(f.Value, func(v string) { - t[CanonicalHeaderKey(v)] = nil - }) - } else { - rl.nextRes.Header.Add(key, f.Value) - } -} - -func (rl *http2clientConnReadLoop) onNewTrailerField(cs *http2clientStream, f hpack.HeaderField) { - if http2VerboseLogs { - rl.cc.logf("http2: Transport decoded trailer %v", f) - } - if !rl.checkHeaderField(f) { - return - } - if strings.HasPrefix(f.Name, ":") { - - rl.reqMalformed = http2errPseudoTrailers - return - } - - key := CanonicalHeaderKey(f.Name) - - // The spec says one must predeclare their trailers but in practice - // popular users (which is to say the only user we found) do not so we - // violate the spec and accept all of them. - const acceptAllTrailers = true - if _, ok := (*cs.resTrailer)[key]; ok || acceptAllTrailers { - if cs.trailer == nil { - cs.trailer = make(Header) - } - cs.trailer[key] = append(cs.trailer[key], f.Value) - } -} - func (cc *http2ClientConn) logf(format string, args ...interface{}) { cc.t.logf(format, args...) } @@ -6167,6 +6487,79 @@ type http2errorReader struct{ err error } func (r http2errorReader) Read(p []byte) (int, error) { return 0, r.err } +// bodyWriterState encapsulates various state around the Transport's writing +// of the request body, particularly regarding doing delayed writes of the body +// when the request contains "Expect: 100-continue". +type http2bodyWriterState struct { + cs *http2clientStream + timer *time.Timer // if non-nil, we're doing a delayed write + fnonce *sync.Once // to call fn with + fn func() // the code to run in the goroutine, writing the body + resc chan error // result of fn's execution + delay time.Duration // how long we should delay a delayed write for +} + +func (t *http2Transport) getBodyWriterState(cs *http2clientStream, body io.Reader) (s http2bodyWriterState) { + s.cs = cs + if body == nil { + return + } + resc := make(chan error, 1) + s.resc = resc + s.fn = func() { + resc <- cs.writeRequestBody(body, cs.req.Body) + } + s.delay = t.expectContinueTimeout() + if s.delay == 0 || + !httplex.HeaderValuesContainsToken( + cs.req.Header["Expect"], + "100-continue") { + return + } + s.fnonce = new(sync.Once) + + // Arm the timer with a very large duration, which we'll + // intentionally lower later. It has to be large now because + // we need a handle to it before writing the headers, but the + // s.delay value is defined to not start until after the + // request headers were written. + const hugeDuration = 365 * 24 * time.Hour + s.timer = time.AfterFunc(hugeDuration, func() { + s.fnonce.Do(s.fn) + }) + return +} + +func (s http2bodyWriterState) cancel() { + if s.timer != nil { + s.timer.Stop() + } +} + +func (s http2bodyWriterState) on100() { + if s.timer == nil { + + return + } + s.timer.Stop() + go func() { s.fnonce.Do(s.fn) }() +} + +// scheduleBodyWrite starts writing the body, either immediately (in +// the common case) or after the delay timeout. It should not be +// called until after the headers have been written. +func (s http2bodyWriterState) scheduleBodyWrite() { + if s.timer == nil { + + go s.fn() + return + } + http2traceWait100Continue(s.cs.trace) + if s.timer.Stop() { + s.timer.Reset(s.delay) + } +} + // writeFramer is implemented by any type that is used to write frames. type http2writeFramer interface { writeFrame(http2writeContext) error @@ -6380,24 +6773,22 @@ func (wu http2writeWindowUpdate) writeFrame(ctx http2writeContext) error { } func http2encodeHeaders(enc *hpack.Encoder, h Header, keys []string) { - if keys == nil { - keys = make([]string, 0, len(h)) - for k := range h { - keys = append(keys, k) - } - sort.Strings(keys) + sorter := http2sorterPool.Get().(*http2sorter) + + defer http2sorterPool.Put(sorter) + keys = sorter.Keys(h) } for _, k := range keys { vv := h[k] k = http2lowerHeader(k) - if !http2validHeaderFieldName(k) { + if !http2validWireHeaderFieldName(k) { continue } isTE := k == "transfer-encoding" for _, v := range vv { - if !http2validHeaderFieldValue(v) { + if !httplex.ValidHeaderFieldValue(v) { continue } diff --git a/libgo/go/net/http/header.go b/libgo/go/net/http/header.go index 049f32f..6343165 100644 --- a/libgo/go/net/http/header.go +++ b/libgo/go/net/http/header.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -25,7 +25,7 @@ func (h Header) Add(key, value string) { } // Set sets the header entries associated with key to -// the single element value. It replaces any existing +// the single element value. It replaces any existing // values associated with key. func (h Header) Set(key, value string) { textproto.MIMEHeader(h).Set(key, value) @@ -164,9 +164,9 @@ func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) error { } // CanonicalHeaderKey returns the canonical format of the -// header key s. The canonicalization converts the first +// header key s. The canonicalization converts the first // letter and any letter following a hyphen to upper case; -// the rest are converted to lowercase. For example, the +// the rest are converted to lowercase. For example, the // canonical key for "accept-encoding" is "Accept-Encoding". // If s contains a space or invalid header field bytes, it is // returned without modifications. @@ -186,7 +186,7 @@ func hasToken(v, token string) bool { for sp := 0; sp <= len(v)-len(token); sp++ { // Check that first character is good. // The token is ASCII, so checking only a single byte - // is sufficient. We skip this potential starting + // is sufficient. We skip this potential starting // position if both the first byte and its potential // ASCII uppercase equivalent (b|0x20) don't match. // False positives ('^' => '~') are caught by EqualFold. diff --git a/libgo/go/net/http/header_test.go b/libgo/go/net/http/header_test.go index 299576b..5c0de15 100644 --- a/libgo/go/net/http/header_test.go +++ b/libgo/go/net/http/header_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/http/http.go b/libgo/go/net/http/http.go new file mode 100644 index 0000000..b34ae41 --- /dev/null +++ b/libgo/go/net/http/http.go @@ -0,0 +1,43 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "strings" + + "golang_org/x/net/lex/httplex" +) + +// maxInt64 is the effective "infinite" value for the Server and +// Transport's byte-limiting readers. +const maxInt64 = 1<<63 - 1 + +// TODO(bradfitz): move common stuff here. The other files have accumulated +// generic http stuff in random places. + +// contextKey is a value for use with context.WithValue. It's used as +// a pointer so it fits in an interface{} without allocation. +type contextKey struct { + name string +} + +func (k *contextKey) String() string { return "net/http context value " + k.name } + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +// removeEmptyPort strips the empty port in ":port" to "" +// as mandated by RFC 3986 Section 6.2.3. +func removeEmptyPort(host string) string { + if hasPort(host) { + return strings.TrimSuffix(host, ":") + } + return host +} + +func isNotToken(r rune) bool { + return !httplex.IsTokenRune(r) +} diff --git a/libgo/go/net/http/http_test.go b/libgo/go/net/http/http_test.go index dead3b0..34da4bb 100644 --- a/libgo/go/net/http/http_test.go +++ b/libgo/go/net/http/http_test.go @@ -2,11 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Tests of internal functions with no better homes. +// Tests of internal functions and things with no better homes. package http import ( + "bytes" + "internal/testenv" + "os/exec" "reflect" "testing" ) @@ -56,3 +59,35 @@ func TestCleanHost(t *testing.T) { } } } + +// Test that cmd/go doesn't link in the HTTP server. +// +// This catches accidental dependencies between the HTTP transport and +// server code. +func TestCmdGoNoHTTPServer(t *testing.T) { + goBin := testenv.GoToolPath(t) + out, err := exec.Command("go", "tool", "nm", goBin).CombinedOutput() + if err != nil { + t.Fatalf("go tool nm: %v: %s", err, out) + } + wantSym := map[string]bool{ + // Verify these exist: (sanity checking this test) + "net/http.(*Client).Get": true, + "net/http.(*Transport).RoundTrip": true, + + // Verify these don't exist: + "net/http.http2Server": false, + "net/http.(*Server).Serve": false, + "net/http.(*ServeMux).ServeHTTP": false, + "net/http.DefaultServeMux": false, + } + for sym, want := range wantSym { + got := bytes.Contains(out, []byte(sym)) + if !want && got { + t.Errorf("cmd/go unexpectedly links in HTTP server code; found symbol %q in cmd/go", sym) + } + if want && !got { + t.Errorf("expected to find symbol %q in cmd/go; not found", sym) + } + } +} diff --git a/libgo/go/net/http/httptest/httptest.go b/libgo/go/net/http/httptest/httptest.go new file mode 100644 index 0000000..e2148a6 --- /dev/null +++ b/libgo/go/net/http/httptest/httptest.go @@ -0,0 +1,88 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httptest provides utilities for HTTP testing. +package httptest + +import ( + "bufio" + "bytes" + "crypto/tls" + "io" + "io/ioutil" + "net/http" + "strings" +) + +// NewRequest returns a new incoming server Request, suitable +// for passing to an http.Handler for testing. +// +// The target is the RFC 7230 "request-target": it may be either a +// path or an absolute URL. If target is an absolute URL, the host name +// from the URL is used. Otherwise, "example.com" is used. +// +// The TLS field is set to a non-nil dummy value if target has scheme +// "https". +// +// The Request.Proto is always HTTP/1.1. +// +// An empty method means "GET". +// +// The provided body may be nil. If the body is of type *bytes.Reader, +// *strings.Reader, or *bytes.Buffer, the Request.ContentLength is +// set. +// +// NewRequest panics on error for ease of use in testing, where a +// panic is acceptable. +func NewRequest(method, target string, body io.Reader) *http.Request { + if method == "" { + method = "GET" + } + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(method + " " + target + " HTTP/1.0\r\n\r\n"))) + if err != nil { + panic("invalid NewRequest arguments; " + err.Error()) + } + + // HTTP/1.0 was used above to avoid needing a Host field. Change it to 1.1 here. + req.Proto = "HTTP/1.1" + req.ProtoMinor = 1 + req.Close = false + + if body != nil { + switch v := body.(type) { + case *bytes.Buffer: + req.ContentLength = int64(v.Len()) + case *bytes.Reader: + req.ContentLength = int64(v.Len()) + case *strings.Reader: + req.ContentLength = int64(v.Len()) + default: + req.ContentLength = -1 + } + if rc, ok := body.(io.ReadCloser); ok { + req.Body = rc + } else { + req.Body = ioutil.NopCloser(body) + } + } + + // 192.0.2.0/24 is "TEST-NET" in RFC 5737 for use solely in + // documentation and example source code and should not be + // used publicly. + req.RemoteAddr = "192.0.2.1:1234" + + if req.Host == "" { + req.Host = "example.com" + } + + if strings.HasPrefix(target, "https://") { + req.TLS = &tls.ConnectionState{ + Version: tls.VersionTLS12, + HandshakeComplete: true, + ServerName: req.Host, + } + } + + return req +} diff --git a/libgo/go/net/http/httptest/httptest_test.go b/libgo/go/net/http/httptest/httptest_test.go new file mode 100644 index 0000000..4f9ecbd --- /dev/null +++ b/libgo/go/net/http/httptest/httptest_test.go @@ -0,0 +1,177 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httptest + +import ( + "crypto/tls" + "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + "strings" + "testing" +) + +func TestNewRequest(t *testing.T) { + tests := [...]struct { + method, uri string + body io.Reader + + want *http.Request + wantBody string + }{ + // Empty method means GET: + 0: { + method: "", + uri: "/", + body: nil, + want: &http.Request{ + Method: "GET", + Host: "example.com", + URL: &url.URL{Path: "/"}, + Header: http.Header{}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "/", + }, + wantBody: "", + }, + + // GET with full URL: + 1: { + method: "GET", + uri: "http://foo.com/path/%2f/bar/", + body: nil, + want: &http.Request{ + Method: "GET", + Host: "foo.com", + URL: &url.URL{ + Scheme: "http", + Path: "/path///bar/", + RawPath: "/path/%2f/bar/", + Host: "foo.com", + }, + Header: http.Header{}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "http://foo.com/path/%2f/bar/", + }, + wantBody: "", + }, + + // GET with full https URL: + 2: { + method: "GET", + uri: "https://foo.com/path/", + body: nil, + want: &http.Request{ + Method: "GET", + Host: "foo.com", + URL: &url.URL{ + Scheme: "https", + Path: "/path/", + Host: "foo.com", + }, + Header: http.Header{}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "https://foo.com/path/", + TLS: &tls.ConnectionState{ + Version: tls.VersionTLS12, + HandshakeComplete: true, + ServerName: "foo.com", + }, + }, + wantBody: "", + }, + + // Post with known length + 3: { + method: "POST", + uri: "/", + body: strings.NewReader("foo"), + want: &http.Request{ + Method: "POST", + Host: "example.com", + URL: &url.URL{Path: "/"}, + Header: http.Header{}, + Proto: "HTTP/1.1", + ContentLength: 3, + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "/", + }, + wantBody: "foo", + }, + + // Post with unknown length + 4: { + method: "POST", + uri: "/", + body: struct{ io.Reader }{strings.NewReader("foo")}, + want: &http.Request{ + Method: "POST", + Host: "example.com", + URL: &url.URL{Path: "/"}, + Header: http.Header{}, + Proto: "HTTP/1.1", + ContentLength: -1, + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "/", + }, + wantBody: "foo", + }, + + // OPTIONS * + 5: { + method: "OPTIONS", + uri: "*", + want: &http.Request{ + Method: "OPTIONS", + Host: "example.com", + URL: &url.URL{Path: "*"}, + Header: http.Header{}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "*", + }, + }, + } + for i, tt := range tests { + got := NewRequest(tt.method, tt.uri, tt.body) + slurp, err := ioutil.ReadAll(got.Body) + if err != nil { + t.Errorf("%d. ReadAll: %v", i, err) + } + if string(slurp) != tt.wantBody { + t.Errorf("%d. Body = %q; want %q", i, slurp, tt.wantBody) + } + got.Body = nil // before DeepEqual + if !reflect.DeepEqual(got.URL, tt.want.URL) { + t.Errorf("%d. Request.URL mismatch:\n got: %#v\nwant: %#v", i, got.URL, tt.want.URL) + } + if !reflect.DeepEqual(got.Header, tt.want.Header) { + t.Errorf("%d. Request.Header mismatch:\n got: %#v\nwant: %#v", i, got.Header, tt.want.Header) + } + if !reflect.DeepEqual(got.TLS, tt.want.TLS) { + t.Errorf("%d. Request.TLS mismatch:\n got: %#v\nwant: %#v", i, got.TLS, tt.want.TLS) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("%d. Request mismatch:\n got: %#v\nwant: %#v", i, got, tt.want) + } + } +} diff --git a/libgo/go/net/http/httptest/recorder.go b/libgo/go/net/http/httptest/recorder.go index 7c51af1..0ad26a3 100644 --- a/libgo/go/net/http/httptest/recorder.go +++ b/libgo/go/net/http/httptest/recorder.go @@ -2,11 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package httptest provides utilities for HTTP testing. package httptest import ( "bytes" + "io/ioutil" "net/http" ) @@ -18,6 +18,8 @@ type ResponseRecorder struct { Body *bytes.Buffer // if non-nil, the bytes.Buffer to append written data to Flushed bool + result *http.Response // cache of Result's return value + snapHeader http.Header // snapshot of HeaderMap at first Write wroteHeader bool } @@ -59,16 +61,15 @@ func (rw *ResponseRecorder) writeHeader(b []byte, str string) { str = str[:512] } - _, hasType := rw.HeaderMap["Content-Type"] - hasTE := rw.HeaderMap.Get("Transfer-Encoding") != "" + m := rw.Header() + + _, hasType := m["Content-Type"] + hasTE := m.Get("Transfer-Encoding") != "" if !hasType && !hasTE { if b == nil { b = []byte(str) } - if rw.HeaderMap == nil { - rw.HeaderMap = make(http.Header) - } - rw.HeaderMap.Set("Content-Type", http.DetectContentType(b)) + m.Set("Content-Type", http.DetectContentType(b)) } rw.WriteHeader(200) @@ -92,12 +93,28 @@ func (rw *ResponseRecorder) WriteString(str string) (int, error) { return len(str), nil } -// WriteHeader sets rw.Code. +// WriteHeader sets rw.Code. After it is called, changing rw.Header +// will not affect rw.HeaderMap. func (rw *ResponseRecorder) WriteHeader(code int) { - if !rw.wroteHeader { - rw.Code = code - rw.wroteHeader = true + if rw.wroteHeader { + return + } + rw.Code = code + rw.wroteHeader = true + if rw.HeaderMap == nil { + rw.HeaderMap = make(http.Header) } + rw.snapHeader = cloneHeader(rw.HeaderMap) +} + +func cloneHeader(h http.Header) http.Header { + h2 := make(http.Header, len(h)) + for k, vv := range h { + vv2 := make([]string, len(vv)) + copy(vv2, vv) + h2[k] = vv2 + } + return h2 } // Flush sets rw.Flushed to true. @@ -107,3 +124,62 @@ func (rw *ResponseRecorder) Flush() { } rw.Flushed = true } + +// Result returns the response generated by the handler. +// +// The returned Response will have at least its StatusCode, +// Header, Body, and optionally Trailer populated. +// More fields may be populated in the future, so callers should +// not DeepEqual the result in tests. +// +// The Response.Header is a snapshot of the headers at the time of the +// first write call, or at the time of this call, if the handler never +// did a write. +// +// Result must only be called after the handler has finished running. +func (rw *ResponseRecorder) Result() *http.Response { + if rw.result != nil { + return rw.result + } + if rw.snapHeader == nil { + rw.snapHeader = cloneHeader(rw.HeaderMap) + } + res := &http.Response{ + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + StatusCode: rw.Code, + Header: rw.snapHeader, + } + rw.result = res + if res.StatusCode == 0 { + res.StatusCode = 200 + } + res.Status = http.StatusText(res.StatusCode) + if rw.Body != nil { + res.Body = ioutil.NopCloser(bytes.NewReader(rw.Body.Bytes())) + } + + if trailers, ok := rw.snapHeader["Trailer"]; ok { + res.Trailer = make(http.Header, len(trailers)) + for _, k := range trailers { + // TODO: use http2.ValidTrailerHeader, but we can't + // get at it easily because it's bundled into net/http + // unexported. This is good enough for now: + switch k { + case "Transfer-Encoding", "Content-Length", "Trailer": + // Ignore since forbidden by RFC 2616 14.40. + continue + } + k = http.CanonicalHeaderKey(k) + vv, ok := rw.HeaderMap[k] + if !ok { + continue + } + vv2 := make([]string, len(vv)) + copy(vv2, vv) + res.Trailer[k] = vv2 + } + } + return res +} diff --git a/libgo/go/net/http/httptest/recorder_test.go b/libgo/go/net/http/httptest/recorder_test.go index c29b6d4..d4e7137 100644 --- a/libgo/go/net/http/httptest/recorder_test.go +++ b/libgo/go/net/http/httptest/recorder_test.go @@ -23,6 +23,14 @@ func TestRecorder(t *testing.T) { return nil } } + hasResultStatus := func(wantCode int) checkFunc { + return func(rec *ResponseRecorder) error { + if rec.Result().StatusCode != wantCode { + return fmt.Errorf("Result().StatusCode = %d; want %d", rec.Result().StatusCode, wantCode) + } + return nil + } + } hasContents := func(want string) checkFunc { return func(rec *ResponseRecorder) error { if rec.Body.String() != want { @@ -39,10 +47,49 @@ func TestRecorder(t *testing.T) { return nil } } - hasHeader := func(key, want string) checkFunc { + hasOldHeader := func(key, want string) checkFunc { return func(rec *ResponseRecorder) error { if got := rec.HeaderMap.Get(key); got != want { - return fmt.Errorf("header %s = %q; want %q", key, got, want) + return fmt.Errorf("HeaderMap header %s = %q; want %q", key, got, want) + } + return nil + } + } + hasHeader := func(key, want string) checkFunc { + return func(rec *ResponseRecorder) error { + if got := rec.Result().Header.Get(key); got != want { + return fmt.Errorf("final header %s = %q; want %q", key, got, want) + } + return nil + } + } + hasNotHeaders := func(keys ...string) checkFunc { + return func(rec *ResponseRecorder) error { + for _, k := range keys { + v, ok := rec.Result().Header[http.CanonicalHeaderKey(k)] + if ok { + return fmt.Errorf("unexpected header %s with value %q", k, v) + } + } + return nil + } + } + hasTrailer := func(key, want string) checkFunc { + return func(rec *ResponseRecorder) error { + if got := rec.Result().Trailer.Get(key); got != want { + return fmt.Errorf("trailer %s = %q; want %q", key, got, want) + } + return nil + } + } + hasNotTrailers := func(keys ...string) checkFunc { + return func(rec *ResponseRecorder) error { + trailers := rec.Result().Trailer + for _, k := range keys { + _, ok := trailers[http.CanonicalHeaderKey(k)] + if ok { + return fmt.Errorf("unexpected trailer %s", k) + } } return nil } @@ -130,6 +177,73 @@ func TestRecorder(t *testing.T) { }, check(hasHeader("Content-Type", "text/html; charset=utf-8")), }, + { + "Header is not changed after write", + func(w http.ResponseWriter, r *http.Request) { + hdr := w.Header() + hdr.Set("Key", "correct") + w.WriteHeader(200) + hdr.Set("Key", "incorrect") + }, + check(hasHeader("Key", "correct")), + }, + { + "Trailer headers are correctly recorded", + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Non-Trailer", "correct") + w.Header().Set("Trailer", "Trailer-A") + w.Header().Add("Trailer", "Trailer-B") + w.Header().Add("Trailer", "Trailer-C") + io.WriteString(w, "<html>") + w.Header().Set("Non-Trailer", "incorrect") + w.Header().Set("Trailer-A", "valuea") + w.Header().Set("Trailer-C", "valuec") + w.Header().Set("Trailer-NotDeclared", "should be omitted") + }, + check( + hasStatus(200), + hasHeader("Content-Type", "text/html; charset=utf-8"), + hasHeader("Non-Trailer", "correct"), + hasNotHeaders("Trailer-A", "Trailer-B", "Trailer-C", "Trailer-NotDeclared"), + hasTrailer("Trailer-A", "valuea"), + hasTrailer("Trailer-C", "valuec"), + hasNotTrailers("Non-Trailer", "Trailer-B", "Trailer-NotDeclared"), + ), + }, + { + "Header set without any write", // Issue 15560 + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Foo", "1") + + // Simulate somebody using + // new(ResponseRecorder) instead of + // using the constructor which sets + // this to 200 + w.(*ResponseRecorder).Code = 0 + }, + check( + hasOldHeader("X-Foo", "1"), + hasStatus(0), + hasHeader("X-Foo", "1"), + hasResultStatus(200), + ), + }, + { + "HeaderMap vs FinalHeaders", // more for Issue 15560 + func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("X-Foo", "1") + w.Write([]byte("hi")) + h.Set("X-Foo", "2") + h.Set("X-Bar", "2") + }, + check( + hasOldHeader("X-Foo", "2"), + hasOldHeader("X-Bar", "2"), + hasHeader("X-Foo", "1"), + hasNotHeaders("X-Bar"), + ), + }, } r, _ := http.NewRequest("GET", "http://foo.com/", nil) for _, tt := range tests { diff --git a/libgo/go/net/http/httptest/server.go b/libgo/go/net/http/httptest/server.go index bbe3233..e27526a 100644 --- a/libgo/go/net/http/httptest/server.go +++ b/libgo/go/net/http/httptest/server.go @@ -158,7 +158,7 @@ func (s *Server) Close() { // previously-flaky tests) in the case of // socket-late-binding races from the http Client // dialing this server and then getting an idle - // connection before the dial completed. There is thus + // connection before the dial completed. There is thus // a connected connection in StateNew with no // associated Request. We only close StateIdle and // StateNew because they're not doing anything. It's @@ -167,7 +167,7 @@ func (s *Server) Close() { // few milliseconds wasn't liked (early versions of // https://golang.org/cl/15151) so now we just // forcefully close StateNew. The docs for Server.Close say - // we wait for "oustanding requests", so we don't close things + // we wait for "outstanding requests", so we don't close things // in StateActive. if st == http.StateIdle || st == http.StateNew { s.closeConn(c) @@ -202,12 +202,10 @@ func (s *Server) logCloseHangDebugInfo() { // CloseClientConnections closes any open HTTP connections to the test Server. func (s *Server) CloseClientConnections() { - var conns int - ch := make(chan bool) - s.mu.Lock() + nconn := len(s.conns) + ch := make(chan struct{}, nconn) for c := range s.conns { - conns++ s.closeConnChan(c, ch) } s.mu.Unlock() @@ -220,7 +218,7 @@ func (s *Server) CloseClientConnections() { // in tests. timer := time.NewTimer(5 * time.Second) defer timer.Stop() - for i := 0; i < conns; i++ { + for i := 0; i < nconn; i++ { select { case <-ch: case <-timer.C: @@ -294,30 +292,20 @@ func (s *Server) closeConn(c net.Conn) { s.closeConnChan(c, nil) } // closeConnChan is like closeConn, but takes an optional channel to receive a value // when the goroutine closing c is done. -func (s *Server) closeConnChan(c net.Conn, done chan<- bool) { +func (s *Server) closeConnChan(c net.Conn, done chan<- struct{}) { if runtime.GOOS == "plan9" { // Go's Plan 9 net package isn't great at unblocking reads when - // their underlying TCP connections are closed. Don't trust + // their underlying TCP connections are closed. Don't trust // that that the ConnState state machine will get to // StateClosed. Instead, just go there directly. Plan 9 may leak // resources if the syscall doesn't end up returning. Oh well. s.forgetConn(c) } - // Somewhere in the chaos of https://golang.org/cl/15151 we found that - // some types of conns were blocking in Close too long (or deadlocking?) - // and we had to call Close in a goroutine. I (bradfitz) forget what - // that was at this point, but I suspect it was *tls.Conns, which - // were later fixed in https://golang.org/cl/18572, so this goroutine - // is _probably_ unnecessary now. But it's too late in Go 1.6 too remove - // it with confidence. - // TODO(bradfitz): try to remove it for Go 1.7. (golang.org/issue/14291) - go func() { - c.Close() - if done != nil { - done <- true - } - }() + c.Close() + if done != nil { + done <- struct{}{} + } } // forgetConn removes c from the set of tracked conns and decrements it from the diff --git a/libgo/go/net/http/httptest/server_test.go b/libgo/go/net/http/httptest/server_test.go index c9606f2..d032c59 100644 --- a/libgo/go/net/http/httptest/server_test.go +++ b/libgo/go/net/http/httptest/server_test.go @@ -53,7 +53,7 @@ func TestGetAfterClose(t *testing.T) { res, err = http.Get(ts.URL) if err == nil { body, _ := ioutil.ReadAll(res.Body) - t.Fatalf("Unexected response after close: %v, %v, %s", res.Status, res.Header, body) + t.Fatalf("Unexpected response after close: %v, %v, %s", res.Status, res.Header, body) } } @@ -95,6 +95,6 @@ func TestServerCloseClientConnections(t *testing.T) { res, err := http.Get(s.URL) if err == nil { res.Body.Close() - t.Fatal("Unexpected response: %#v", res) + t.Fatalf("Unexpected response: %#v", res) } } diff --git a/libgo/go/net/http/httptrace/trace.go b/libgo/go/net/http/httptrace/trace.go new file mode 100644 index 0000000..6f187a7 --- /dev/null +++ b/libgo/go/net/http/httptrace/trace.go @@ -0,0 +1,226 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file.h + +// Package httptrace provides mechanisms to trace the events within +// HTTP client requests. +package httptrace + +import ( + "context" + "internal/nettrace" + "net" + "reflect" + "time" +) + +// unique type to prevent assignment. +type clientEventContextKey struct{} + +// ContextClientTrace returns the ClientTrace associated with the +// provided context. If none, it returns nil. +func ContextClientTrace(ctx context.Context) *ClientTrace { + trace, _ := ctx.Value(clientEventContextKey{}).(*ClientTrace) + return trace +} + +// WithClientTrace returns a new context based on the provided parent +// ctx. HTTP client requests made with the returned context will use +// the provided trace hooks, in addition to any previous hooks +// registered with ctx. Any hooks defined in the provided trace will +// be called first. +func WithClientTrace(ctx context.Context, trace *ClientTrace) context.Context { + if trace == nil { + panic("nil trace") + } + old := ContextClientTrace(ctx) + trace.compose(old) + + ctx = context.WithValue(ctx, clientEventContextKey{}, trace) + if trace.hasNetHooks() { + nt := &nettrace.Trace{ + ConnectStart: trace.ConnectStart, + ConnectDone: trace.ConnectDone, + } + if trace.DNSStart != nil { + nt.DNSStart = func(name string) { + trace.DNSStart(DNSStartInfo{Host: name}) + } + } + if trace.DNSDone != nil { + nt.DNSDone = func(netIPs []interface{}, coalesced bool, err error) { + addrs := make([]net.IPAddr, len(netIPs)) + for i, ip := range netIPs { + addrs[i] = ip.(net.IPAddr) + } + trace.DNSDone(DNSDoneInfo{ + Addrs: addrs, + Coalesced: coalesced, + Err: err, + }) + } + } + ctx = context.WithValue(ctx, nettrace.TraceKey{}, nt) + } + return ctx +} + +// ClientTrace is a set of hooks to run at various stages of an HTTP +// client request. Any particular hook may be nil. Functions may be +// called concurrently from different goroutines, starting after the +// call to Transport.RoundTrip and ending either when RoundTrip +// returns an error, or when the Response.Body is closed. +type ClientTrace struct { + // GetConn is called before a connection is created or + // retrieved from an idle pool. The hostPort is the + // "host:port" of the target or proxy. GetConn is called even + // if there's already an idle cached connection available. + GetConn func(hostPort string) + + // GotConn is called after a successful connection is + // obtained. There is no hook for failure to obtain a + // connection; instead, use the error from + // Transport.RoundTrip. + GotConn func(GotConnInfo) + + // PutIdleConn is called when the connection is returned to + // the idle pool. If err is nil, the connection was + // successfully returned to the idle pool. If err is non-nil, + // it describes why not. PutIdleConn is not called if + // connection reuse is disabled via Transport.DisableKeepAlives. + // PutIdleConn is called before the caller's Response.Body.Close + // call returns. + // For HTTP/2, this hook is not currently used. + PutIdleConn func(err error) + + // GotFirstResponseByte is called when the first byte of the response + // headers is available. + GotFirstResponseByte func() + + // Got100Continue is called if the server replies with a "100 + // Continue" response. + Got100Continue func() + + // DNSStart is called when a DNS lookup begins. + DNSStart func(DNSStartInfo) + + // DNSDone is called when a DNS lookup ends. + DNSDone func(DNSDoneInfo) + + // ConnectStart is called when a new connection's Dial begins. + // If net.Dialer.DualStack (IPv6 "Happy Eyeballs") support is + // enabled, this may be called multiple times. + ConnectStart func(network, addr string) + + // ConnectDone is called when a new connection's Dial + // completes. The provided err indicates whether the + // connection completedly successfully. + // If net.Dialer.DualStack ("Happy Eyeballs") support is + // enabled, this may be called multiple times. + ConnectDone func(network, addr string, err error) + + // WroteHeaders is called after the Transport has written + // the request headers. + WroteHeaders func() + + // Wait100Continue is called if the Request specified + // "Expected: 100-continue" and the Transport has written the + // request headers but is waiting for "100 Continue" from the + // server before writing the request body. + Wait100Continue func() + + // WroteRequest is called with the result of writing the + // request and any body. + WroteRequest func(WroteRequestInfo) +} + +// WroteRequestInfo contains information provided to the WroteRequest +// hook. +type WroteRequestInfo struct { + // Err is any error encountered while writing the Request. + Err error +} + +// compose modifies t such that it respects the previously-registered hooks in old, +// subject to the composition policy requested in t.Compose. +func (t *ClientTrace) compose(old *ClientTrace) { + if old == nil { + return + } + tv := reflect.ValueOf(t).Elem() + ov := reflect.ValueOf(old).Elem() + structType := tv.Type() + for i := 0; i < structType.NumField(); i++ { + tf := tv.Field(i) + hookType := tf.Type() + if hookType.Kind() != reflect.Func { + continue + } + of := ov.Field(i) + if of.IsNil() { + continue + } + if tf.IsNil() { + tf.Set(of) + continue + } + + // Make a copy of tf for tf to call. (Otherwise it + // creates a recursive call cycle and stack overflows) + tfCopy := reflect.ValueOf(tf.Interface()) + + // We need to call both tf and of in some order. + newFunc := reflect.MakeFunc(hookType, func(args []reflect.Value) []reflect.Value { + tfCopy.Call(args) + return of.Call(args) + }) + tv.Field(i).Set(newFunc) + } +} + +// DNSStartInfo contains information about a DNS request. +type DNSStartInfo struct { + Host string +} + +// DNSDoneInfo contains information about the results of a DNS lookup. +type DNSDoneInfo struct { + // Addrs are the IPv4 and/or IPv6 addresses found in the DNS + // lookup. The contents of the slice should not be mutated. + Addrs []net.IPAddr + + // Err is any error that occurred during the DNS lookup. + Err error + + // Coalesced is whether the Addrs were shared with another + // caller who was doing the same DNS lookup concurrently. + Coalesced bool +} + +func (t *ClientTrace) hasNetHooks() bool { + if t == nil { + return false + } + return t.DNSStart != nil || t.DNSDone != nil || t.ConnectStart != nil || t.ConnectDone != nil +} + +// GotConnInfo is the argument to the ClientTrace.GotConn function and +// contains information about the obtained connection. +type GotConnInfo struct { + // Conn is the connection that was obtained. It is owned by + // the http.Transport and should not be read, written or + // closed by users of ClientTrace. + Conn net.Conn + + // Reused is whether this connection has been previously + // used for another HTTP request. + Reused bool + + // WasIdle is whether this connection was obtained from an + // idle pool. + WasIdle bool + + // IdleTime reports how long the connection was previously + // idle, if WasIdle is true. + IdleTime time.Duration +} diff --git a/libgo/go/net/http/httptrace/trace_test.go b/libgo/go/net/http/httptrace/trace_test.go new file mode 100644 index 0000000..c7eaed8 --- /dev/null +++ b/libgo/go/net/http/httptrace/trace_test.go @@ -0,0 +1,62 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file.h + +package httptrace + +import ( + "bytes" + "testing" +) + +func TestCompose(t *testing.T) { + var buf bytes.Buffer + var testNum int + + connectStart := func(b byte) func(network, addr string) { + return func(network, addr string) { + if addr != "addr" { + t.Errorf(`%d. args for %q case = %q, %q; want addr of "addr"`, testNum, b, network, addr) + } + buf.WriteByte(b) + } + } + + tests := [...]struct { + trace, old *ClientTrace + want string + }{ + 0: { + want: "T", + trace: &ClientTrace{ + ConnectStart: connectStart('T'), + }, + }, + 1: { + want: "TO", + trace: &ClientTrace{ + ConnectStart: connectStart('T'), + }, + old: &ClientTrace{ConnectStart: connectStart('O')}, + }, + 2: { + want: "O", + trace: &ClientTrace{}, + old: &ClientTrace{ConnectStart: connectStart('O')}, + }, + } + for i, tt := range tests { + testNum = i + buf.Reset() + + tr := *tt.trace + tr.compose(tt.old) + if tr.ConnectStart != nil { + tr.ConnectStart("net", "addr") + } + if got := buf.String(); got != tt.want { + t.Errorf("%d. got = %q; want %q", i, got, tt.want) + } + } + +} diff --git a/libgo/go/net/http/httputil/dump.go b/libgo/go/net/http/httputil/dump.go index e22cc66..1511681 100644 --- a/libgo/go/net/http/httputil/dump.go +++ b/libgo/go/net/http/httputil/dump.go @@ -128,7 +128,7 @@ func DumpRequestOut(req *http.Request, body bool) ([]byte, error) { // If we used a dummy body above, remove it now. // TODO: if the req.ContentLength is large, we allocate memory - // unnecessarily just to slice it off here. But this is just + // unnecessarily just to slice it off here. But this is just // a debug function, so this is acceptable for now. We could // discard the body earlier if this matters. if dummyBody { @@ -163,18 +163,10 @@ func valueOrDefault(value, def string) string { var reqWriteExcludeHeaderDump = map[string]bool{ "Host": true, // not in Header map anyway - "Content-Length": true, "Transfer-Encoding": true, "Trailer": true, } -// dumpAsReceived writes req to w in the form as it was received, or -// at least as accurately as possible from the information retained in -// the request. -func dumpAsReceived(req *http.Request, w io.Writer) error { - return nil -} - // DumpRequest returns the given request in its HTTP/1.x wire // representation. It should only be used by servers to debug client // requests. The returned representation is an approximation only; @@ -191,7 +183,8 @@ func dumpAsReceived(req *http.Request, w io.Writer) error { // // The documentation for http.Request.Write details which fields // of req are included in the dump. -func DumpRequest(req *http.Request, body bool) (dump []byte, err error) { +func DumpRequest(req *http.Request, body bool) ([]byte, error) { + var err error save := req.Body if !body || req.Body == nil { req.Body = nil @@ -239,7 +232,7 @@ func DumpRequest(req *http.Request, body bool) (dump []byte, err error) { err = req.Header.WriteSubset(&b, reqWriteExcludeHeaderDump) if err != nil { - return + return nil, err } io.WriteString(&b, "\r\n") @@ -258,35 +251,42 @@ func DumpRequest(req *http.Request, body bool) (dump []byte, err error) { req.Body = save if err != nil { - return + return nil, err } - dump = b.Bytes() - return + return b.Bytes(), nil } -// errNoBody is a sentinel error value used by failureToReadBody so we can detect -// that the lack of body was intentional. +// errNoBody is a sentinel error value used by failureToReadBody so we +// can detect that the lack of body was intentional. var errNoBody = errors.New("sentinel error value") // failureToReadBody is a io.ReadCloser that just returns errNoBody on -// Read. It's swapped in when we don't actually want to consume the -// body, but need a non-nil one, and want to distinguish the error -// from reading the dummy body. +// Read. It's swapped in when we don't actually want to consume +// the body, but need a non-nil one, and want to distinguish the +// error from reading the dummy body. type failureToReadBody struct{} func (failureToReadBody) Read([]byte) (int, error) { return 0, errNoBody } func (failureToReadBody) Close() error { return nil } +// emptyBody is an instance of empty reader. var emptyBody = ioutil.NopCloser(strings.NewReader("")) // DumpResponse is like DumpRequest but dumps a response. -func DumpResponse(resp *http.Response, body bool) (dump []byte, err error) { +func DumpResponse(resp *http.Response, body bool) ([]byte, error) { var b bytes.Buffer + var err error save := resp.Body savecl := resp.ContentLength if !body { - resp.Body = failureToReadBody{} + // For content length of zero. Make sure the body is an empty + // reader, instead of returning error through failureToReadBody{}. + if resp.ContentLength == 0 { + resp.Body = emptyBody + } else { + resp.Body = failureToReadBody{} + } } else if resp.Body == nil { resp.Body = emptyBody } else { diff --git a/libgo/go/net/http/httputil/dump_test.go b/libgo/go/net/http/httputil/dump_test.go index 46bf521..2e980d3 100644 --- a/libgo/go/net/http/httputil/dump_test.go +++ b/libgo/go/net/http/httputil/dump_test.go @@ -122,6 +122,10 @@ var dumpTests = []dumpTest{ Host: "post.tld", Path: "/", }, + Header: http.Header{ + "Content-Length": []string{"8193"}, + }, + ContentLength: 8193, ProtoMajor: 1, ProtoMinor: 1, @@ -135,6 +139,10 @@ var dumpTests = []dumpTest{ "Content-Length: 8193\r\n" + "Accept-Encoding: gzip\r\n\r\n" + strings.Repeat("a", 8193), + WantDump: "POST / HTTP/1.1\r\n" + + "Host: post.tld\r\n" + + "Content-Length: 8193\r\n\r\n" + + strings.Repeat("a", 8193), }, { @@ -144,6 +152,38 @@ var dumpTests = []dumpTest{ WantDump: "GET http://foo.com/ HTTP/1.1\r\n" + "User-Agent: blah\r\n\r\n", }, + + // Issue #7215. DumpRequest should return the "Content-Length" when set + { + Req: *mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 3\r\n" + + "\r\nkey1=name1&key2=name2"), + WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 3\r\n" + + "\r\nkey", + }, + + // Issue #7215. DumpRequest should return the "Content-Length" in ReadRequest + { + Req: *mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 0\r\n" + + "\r\nkey1=name1&key2=name2"), + WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 0\r\n\r\n", + }, + + // Issue #7215. DumpRequest should not return the "Content-Length" if unset + { + Req: *mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "\r\nkey1=name1&key2=name2"), + WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n\r\n", + }, } func TestDumpRequest(t *testing.T) { @@ -288,6 +328,27 @@ Transfer-Encoding: chunked foo 0`, }, + { + res: &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 0, + Header: http.Header{ + // To verify if headers are not filtered out. + "Foo1": []string{"Bar1"}, + "Foo2": []string{"Bar2"}, + }, + Body: nil, + }, + body: false, // to verify we see 0, not empty. + want: `HTTP/1.1 200 OK +Foo1: Bar1 +Foo2: Bar2 +Content-Length: 0`, + }, } func TestDumpResponse(t *testing.T) { diff --git a/libgo/go/net/http/httputil/example_test.go b/libgo/go/net/http/httputil/example_test.go index 8fb1a2d..e8dc962 100644 --- a/libgo/go/net/http/httputil/example_test.go +++ b/libgo/go/net/http/httputil/example_test.go @@ -49,7 +49,7 @@ func ExampleDumpRequest() { fmt.Printf("%s", b) // Output: - // "POST / HTTP/1.1\r\nHost: www.example.org\r\nAccept-Encoding: gzip\r\nUser-Agent: Go-http-client/1.1\r\n\r\nGo is a general-purpose language designed with systems programming in mind." + // "POST / HTTP/1.1\r\nHost: www.example.org\r\nAccept-Encoding: gzip\r\nContent-Length: 75\r\nUser-Agent: Go-http-client/1.1\r\n\r\nGo is a general-purpose language designed with systems programming in mind." } func ExampleDumpRequestOut() { diff --git a/libgo/go/net/http/httputil/persist.go b/libgo/go/net/http/httputil/persist.go index 987bcc9..87ddd52 100644 --- a/libgo/go/net/http/httputil/persist.go +++ b/libgo/go/net/http/httputil/persist.go @@ -24,17 +24,13 @@ var ( // ErrPersistEOF (above) reports that the remote side is closed. var errClosed = errors.New("i/o operation on closed connection") -// A ServerConn reads requests and sends responses over an underlying -// connection, until the HTTP keepalive logic commands an end. ServerConn -// also allows hijacking the underlying connection by calling Hijack -// to regain control over the connection. ServerConn supports pipe-lining, -// i.e. requests can be read out of sync (but in the same order) while the -// respective responses are sent. +// ServerConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// ServerConn is low-level and old. Applications should instead use Server -// in the net/http package. +// Deprecated: Use the Server in package net/http instead. type ServerConn struct { - lk sync.Mutex // read-write protects the following fields + mu sync.Mutex // read-write protects the following fields c net.Conn r *bufio.Reader re, we error // read/write errors @@ -45,11 +41,11 @@ type ServerConn struct { pipe textproto.Pipeline } -// NewServerConn returns a new ServerConn reading and writing c. If r is not -// nil, it is the buffer to use when reading c. +// NewServerConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// ServerConn is low-level and old. Applications should instead use Server -// in the net/http package. +// Deprecated: Use the Server in package net/http instead. func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn { if r == nil { r = bufio.NewReader(c) @@ -61,17 +57,17 @@ func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn { // as the read-side bufio which may have some left over data. Hijack may be // called before Read has signaled the end of the keep-alive logic. The user // should not call Hijack while Read or Write is in progress. -func (sc *ServerConn) Hijack() (c net.Conn, r *bufio.Reader) { - sc.lk.Lock() - defer sc.lk.Unlock() - c = sc.c - r = sc.r +func (sc *ServerConn) Hijack() (net.Conn, *bufio.Reader) { + sc.mu.Lock() + defer sc.mu.Unlock() + c := sc.c + r := sc.r sc.c = nil sc.r = nil - return + return c, r } -// Close calls Hijack and then also closes the underlying connection +// Close calls Hijack and then also closes the underlying connection. func (sc *ServerConn) Close() error { c, _ := sc.Hijack() if c != nil { @@ -84,7 +80,9 @@ func (sc *ServerConn) Close() error { // it is gracefully determined that there are no more requests (e.g. after the // first request on an HTTP/1.0 connection, or after a Connection:close on a // HTTP/1.1 connection). -func (sc *ServerConn) Read() (req *http.Request, err error) { +func (sc *ServerConn) Read() (*http.Request, error) { + var req *http.Request + var err error // Ensure ordered execution of Reads and Writes id := sc.pipe.Next() @@ -96,29 +94,29 @@ func (sc *ServerConn) Read() (req *http.Request, err error) { sc.pipe.EndResponse(id) } else { // Remember the pipeline id of this request - sc.lk.Lock() + sc.mu.Lock() sc.pipereq[req] = id - sc.lk.Unlock() + sc.mu.Unlock() } }() - sc.lk.Lock() + sc.mu.Lock() if sc.we != nil { // no point receiving if write-side broken or closed - defer sc.lk.Unlock() + defer sc.mu.Unlock() return nil, sc.we } if sc.re != nil { - defer sc.lk.Unlock() + defer sc.mu.Unlock() return nil, sc.re } if sc.r == nil { // connection closed by user in the meantime - defer sc.lk.Unlock() + defer sc.mu.Unlock() return nil, errClosed } r := sc.r lastbody := sc.lastbody sc.lastbody = nil - sc.lk.Unlock() + sc.mu.Unlock() // Make sure body is fully consumed, even if user does not call body.Close if lastbody != nil { @@ -127,16 +125,16 @@ func (sc *ServerConn) Read() (req *http.Request, err error) { // returned. err = lastbody.Close() if err != nil { - sc.lk.Lock() - defer sc.lk.Unlock() + sc.mu.Lock() + defer sc.mu.Unlock() sc.re = err return nil, err } } req, err = http.ReadRequest(r) - sc.lk.Lock() - defer sc.lk.Unlock() + sc.mu.Lock() + defer sc.mu.Unlock() if err != nil { if err == io.ErrUnexpectedEOF { // A close from the opposing client is treated as a @@ -161,8 +159,8 @@ func (sc *ServerConn) Read() (req *http.Request, err error) { // Pending returns the number of unanswered requests // that have been received on the connection. func (sc *ServerConn) Pending() int { - sc.lk.Lock() - defer sc.lk.Unlock() + sc.mu.Lock() + defer sc.mu.Unlock() return sc.nread - sc.nwritten } @@ -172,31 +170,31 @@ func (sc *ServerConn) Pending() int { func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error { // Retrieve the pipeline ID of this request/response pair - sc.lk.Lock() + sc.mu.Lock() id, ok := sc.pipereq[req] delete(sc.pipereq, req) if !ok { - sc.lk.Unlock() + sc.mu.Unlock() return ErrPipeline } - sc.lk.Unlock() + sc.mu.Unlock() // Ensure pipeline order sc.pipe.StartResponse(id) defer sc.pipe.EndResponse(id) - sc.lk.Lock() + sc.mu.Lock() if sc.we != nil { - defer sc.lk.Unlock() + defer sc.mu.Unlock() return sc.we } if sc.c == nil { // connection closed by user in the meantime - defer sc.lk.Unlock() + defer sc.mu.Unlock() return ErrClosed } c := sc.c if sc.nread <= sc.nwritten { - defer sc.lk.Unlock() + defer sc.mu.Unlock() return errors.New("persist server pipe count") } if resp.Close { @@ -205,11 +203,11 @@ func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error { // before signaling. sc.re = ErrPersistEOF } - sc.lk.Unlock() + sc.mu.Unlock() err := resp.Write(c) - sc.lk.Lock() - defer sc.lk.Unlock() + sc.mu.Lock() + defer sc.mu.Unlock() if err != nil { sc.we = err return err @@ -219,15 +217,13 @@ func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error { return nil } -// A ClientConn sends request and receives headers over an underlying -// connection, while respecting the HTTP keepalive logic. ClientConn -// supports hijacking the connection calling Hijack to -// regain control of the underlying net.Conn and deal with it as desired. +// ClientConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// ClientConn is low-level and old. Applications should instead use -// Client or Transport in the net/http package. +// Deprecated: Use Client or Transport in package net/http instead. type ClientConn struct { - lk sync.Mutex // read-write protects the following fields + mu sync.Mutex // read-write protects the following fields c net.Conn r *bufio.Reader re, we error // read/write errors @@ -239,11 +235,11 @@ type ClientConn struct { writeReq func(*http.Request, io.Writer) error } -// NewClientConn returns a new ClientConn reading and writing c. If r is not -// nil, it is the buffer to use when reading c. +// NewClientConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// ClientConn is low-level and old. Applications should use Client or -// Transport in the net/http package. +// Deprecated: Use the Client or Transport in package net/http instead. func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { if r == nil { r = bufio.NewReader(c) @@ -256,11 +252,11 @@ func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { } } -// NewProxyClientConn works like NewClientConn but writes Requests -// using Request's WriteProxy method. +// NewProxyClientConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// New code should not use NewProxyClientConn. See Client or -// Transport in the net/http package instead. +// Deprecated: Use the Client or Transport in package net/http instead. func NewProxyClientConn(c net.Conn, r *bufio.Reader) *ClientConn { cc := NewClientConn(c, r) cc.writeReq = (*http.Request).WriteProxy @@ -272,8 +268,8 @@ func NewProxyClientConn(c net.Conn, r *bufio.Reader) *ClientConn { // called before the user or Read have signaled the end of the keep-alive // logic. The user should not call Hijack while Read or Write is in progress. func (cc *ClientConn) Hijack() (c net.Conn, r *bufio.Reader) { - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() c = cc.c r = cc.r cc.c = nil @@ -281,7 +277,7 @@ func (cc *ClientConn) Hijack() (c net.Conn, r *bufio.Reader) { return } -// Close calls Hijack and then also closes the underlying connection +// Close calls Hijack and then also closes the underlying connection. func (cc *ClientConn) Close() error { c, _ := cc.Hijack() if c != nil { @@ -295,7 +291,8 @@ func (cc *ClientConn) Close() error { // keepalive connection is logically closed after this request and the opposing // server is informed. An ErrUnexpectedEOF indicates the remote closed the // underlying TCP connection, which is usually considered as graceful close. -func (cc *ClientConn) Write(req *http.Request) (err error) { +func (cc *ClientConn) Write(req *http.Request) error { + var err error // Ensure ordered execution of Writes id := cc.pipe.Next() @@ -307,23 +304,23 @@ func (cc *ClientConn) Write(req *http.Request) (err error) { cc.pipe.EndResponse(id) } else { // Remember the pipeline id of this request - cc.lk.Lock() + cc.mu.Lock() cc.pipereq[req] = id - cc.lk.Unlock() + cc.mu.Unlock() } }() - cc.lk.Lock() + cc.mu.Lock() if cc.re != nil { // no point sending if read-side closed or broken - defer cc.lk.Unlock() + defer cc.mu.Unlock() return cc.re } if cc.we != nil { - defer cc.lk.Unlock() + defer cc.mu.Unlock() return cc.we } if cc.c == nil { // connection closed by user in the meantime - defer cc.lk.Unlock() + defer cc.mu.Unlock() return errClosed } c := cc.c @@ -332,11 +329,11 @@ func (cc *ClientConn) Write(req *http.Request) (err error) { // still might be some pipelined reads cc.we = ErrPersistEOF } - cc.lk.Unlock() + cc.mu.Unlock() err = cc.writeReq(req, c) - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() if err != nil { cc.we = err return err @@ -349,8 +346,8 @@ func (cc *ClientConn) Write(req *http.Request) (err error) { // Pending returns the number of unanswered requests // that have been sent on the connection. func (cc *ClientConn) Pending() int { - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() return cc.nwritten - cc.nread } @@ -360,32 +357,32 @@ func (cc *ClientConn) Pending() int { // concurrently with Write, but not with another Read. func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) { // Retrieve the pipeline ID of this request/response pair - cc.lk.Lock() + cc.mu.Lock() id, ok := cc.pipereq[req] delete(cc.pipereq, req) if !ok { - cc.lk.Unlock() + cc.mu.Unlock() return nil, ErrPipeline } - cc.lk.Unlock() + cc.mu.Unlock() // Ensure pipeline order cc.pipe.StartResponse(id) defer cc.pipe.EndResponse(id) - cc.lk.Lock() + cc.mu.Lock() if cc.re != nil { - defer cc.lk.Unlock() + defer cc.mu.Unlock() return nil, cc.re } if cc.r == nil { // connection closed by user in the meantime - defer cc.lk.Unlock() + defer cc.mu.Unlock() return nil, errClosed } r := cc.r lastbody := cc.lastbody cc.lastbody = nil - cc.lk.Unlock() + cc.mu.Unlock() // Make sure body is fully consumed, even if user does not call body.Close if lastbody != nil { @@ -394,16 +391,16 @@ func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) { // returned. err = lastbody.Close() if err != nil { - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() cc.re = err return nil, err } } resp, err = http.ReadResponse(r, req) - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() if err != nil { cc.re = err return resp, err @@ -420,10 +417,10 @@ func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) { } // Do is convenience method that writes a request and reads a response. -func (cc *ClientConn) Do(req *http.Request) (resp *http.Response, err error) { - err = cc.Write(req) +func (cc *ClientConn) Do(req *http.Request) (*http.Response, error) { + err := cc.Write(req) if err != nil { - return + return nil, err } return cc.Read(req) } diff --git a/libgo/go/net/http/httputil/reverseproxy.go b/libgo/go/net/http/httputil/reverseproxy.go index 54411ca..49c120a 100644 --- a/libgo/go/net/http/httputil/reverseproxy.go +++ b/libgo/go/net/http/httputil/reverseproxy.go @@ -90,6 +90,10 @@ func NewSingleHostReverseProxy(target *url.URL) *ReverseProxy { } else { req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery } + if _, ok := req.Header["User-Agent"]; !ok { + // explicitly disable User-Agent so it's not set to default value + req.Header.Set("User-Agent", "") + } } return &ReverseProxy{Director: director} } @@ -180,9 +184,9 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { outreq.ProtoMinor = 1 outreq.Close = false - // Remove hop-by-hop headers to the backend. Especially + // Remove hop-by-hop headers to the backend. Especially // important is "Connection" because we want a persistent - // connection, regardless of what the client sent to us. This + // connection, regardless of what the client sent to us. This // is modifying the same underlying map from req (shallow // copied above) so we only copy it if necessary. copiedHeaders := false @@ -210,7 +214,7 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { res, err := transport.RoundTrip(outreq) if err != nil { p.logf("http: proxy error: %v", err) - rw.WriteHeader(http.StatusInternalServerError) + rw.WriteHeader(http.StatusBadGateway) return } @@ -285,13 +289,13 @@ type maxLatencyWriter struct { dst writeFlusher latency time.Duration - lk sync.Mutex // protects Write + Flush + mu sync.Mutex // protects Write + Flush done chan bool } func (m *maxLatencyWriter) Write(p []byte) (int, error) { - m.lk.Lock() - defer m.lk.Unlock() + m.mu.Lock() + defer m.mu.Unlock() return m.dst.Write(p) } @@ -306,9 +310,9 @@ func (m *maxLatencyWriter) flushLoop() { } return case <-t.C: - m.lk.Lock() + m.mu.Lock() m.dst.Flush() - m.lk.Unlock() + m.mu.Unlock() } } } diff --git a/libgo/go/net/http/httputil/reverseproxy_test.go b/libgo/go/net/http/httputil/reverseproxy_test.go index 0849427..fe7cdb8 100644 --- a/libgo/go/net/http/httputil/reverseproxy_test.go +++ b/libgo/go/net/http/httputil/reverseproxy_test.go @@ -33,6 +33,11 @@ func TestReverseProxy(t *testing.T) { const backendResponse = "I am the backend" const backendStatus = 404 backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" && r.FormValue("mode") == "hangup" { + c, _, _ := w.(http.Hijacker).Hijack() + c.Close() + return + } if len(r.TransferEncoding) > 0 { t.Errorf("backend got unexpected TransferEncoding: %v", r.TransferEncoding) } @@ -69,6 +74,7 @@ func TestReverseProxy(t *testing.T) { t.Fatal(err) } proxyHandler := NewSingleHostReverseProxy(backendURL) + proxyHandler.ErrorLog = log.New(ioutil.Discard, "", 0) // quiet for tests frontend := httptest.NewServer(proxyHandler) defer frontend.Close() @@ -113,6 +119,20 @@ func TestReverseProxy(t *testing.T) { if g, e := res.Trailer.Get("X-Trailer"), "trailer_value"; g != e { t.Errorf("Trailer(X-Trailer) = %q ; want %q", g, e) } + + // Test that a backend failing to be reached or one which doesn't return + // a response results in a StatusBadGateway. + getReq, _ = http.NewRequest("GET", frontend.URL+"/?mode=hangup", nil) + getReq.Close = true + res, err = http.DefaultClient.Do(getReq) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != http.StatusBadGateway { + t.Errorf("request to bad proxy = %v; want 502 StatusBadGateway", res.Status) + } + } func TestXForwardedFor(t *testing.T) { @@ -328,6 +348,49 @@ func TestNilBody(t *testing.T) { } } +// Issue 15524 +func TestUserAgentHeader(t *testing.T) { + const explicitUA = "explicit UA" + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/noua" { + if c := r.Header.Get("User-Agent"); c != "" { + t.Errorf("handler got non-empty User-Agent header %q", c) + } + return + } + if c := r.Header.Get("User-Agent"); c != explicitUA { + t.Errorf("handler got unexpected User-Agent header %q", c) + } + })) + defer backend.Close() + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + proxyHandler := NewSingleHostReverseProxy(backendURL) + proxyHandler.ErrorLog = log.New(ioutil.Discard, "", 0) // quiet for tests + frontend := httptest.NewServer(proxyHandler) + defer frontend.Close() + + getReq, _ := http.NewRequest("GET", frontend.URL, nil) + getReq.Header.Set("User-Agent", explicitUA) + getReq.Close = true + res, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatalf("Get: %v", err) + } + res.Body.Close() + + getReq, _ = http.NewRequest("GET", frontend.URL+"/noua", nil) + getReq.Header.Set("User-Agent", "") + getReq.Close = true + res, err = http.DefaultClient.Do(getReq) + if err != nil { + t.Fatalf("Get: %v", err) + } + res.Body.Close() +} + type bufferPool struct { get func() []byte put func([]byte) diff --git a/libgo/go/net/http/internal/chunked_test.go b/libgo/go/net/http/internal/chunked_test.go index a136dc9..9abe1ab 100644 --- a/libgo/go/net/http/internal/chunked_test.go +++ b/libgo/go/net/http/internal/chunked_test.go @@ -122,7 +122,7 @@ func TestChunkReaderAllocs(t *testing.T) { byter := bytes.NewReader(buf.Bytes()) bufr := bufio.NewReader(byter) mallocs := testing.AllocsPerRun(100, func() { - byter.Seek(0, 0) + byter.Seek(0, io.SeekStart) bufr.Reset(byter) r := NewChunkedReader(bufr) n, err := io.ReadFull(r, readBuf) diff --git a/libgo/go/net/http/lex.go b/libgo/go/net/http/lex.go deleted file mode 100644 index 52b6481..0000000 --- a/libgo/go/net/http/lex.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package http - -import ( - "strings" - "unicode/utf8" -) - -// This file deals with lexical matters of HTTP - -var isTokenTable = [127]bool{ - '!': true, - '#': true, - '$': true, - '%': true, - '&': true, - '\'': true, - '*': true, - '+': true, - '-': true, - '.': true, - '0': true, - '1': true, - '2': true, - '3': true, - '4': true, - '5': true, - '6': true, - '7': true, - '8': true, - '9': true, - 'A': true, - 'B': true, - 'C': true, - 'D': true, - 'E': true, - 'F': true, - 'G': true, - 'H': true, - 'I': true, - 'J': true, - 'K': true, - 'L': true, - 'M': true, - 'N': true, - 'O': true, - 'P': true, - 'Q': true, - 'R': true, - 'S': true, - 'T': true, - 'U': true, - 'W': true, - 'V': true, - 'X': true, - 'Y': true, - 'Z': true, - '^': true, - '_': true, - '`': true, - 'a': true, - 'b': true, - 'c': true, - 'd': true, - 'e': true, - 'f': true, - 'g': true, - 'h': true, - 'i': true, - 'j': true, - 'k': true, - 'l': true, - 'm': true, - 'n': true, - 'o': true, - 'p': true, - 'q': true, - 'r': true, - 's': true, - 't': true, - 'u': true, - 'v': true, - 'w': true, - 'x': true, - 'y': true, - 'z': true, - '|': true, - '~': true, -} - -func isToken(r rune) bool { - i := int(r) - return i < len(isTokenTable) && isTokenTable[i] -} - -func isNotToken(r rune) bool { - return !isToken(r) -} - -// headerValuesContainsToken reports whether any string in values -// contains the provided token, ASCII case-insensitively. -func headerValuesContainsToken(values []string, token string) bool { - for _, v := range values { - if headerValueContainsToken(v, token) { - return true - } - } - return false -} - -// isOWS reports whether b is an optional whitespace byte, as defined -// by RFC 7230 section 3.2.3. -func isOWS(b byte) bool { return b == ' ' || b == '\t' } - -// trimOWS returns x with all optional whitespace removes from the -// beginning and end. -func trimOWS(x string) string { - // TODO: consider using strings.Trim(x, " \t") instead, - // if and when it's fast enough. See issue 10292. - // But this ASCII-only code will probably always beat UTF-8 - // aware code. - for len(x) > 0 && isOWS(x[0]) { - x = x[1:] - } - for len(x) > 0 && isOWS(x[len(x)-1]) { - x = x[:len(x)-1] - } - return x -} - -// headerValueContainsToken reports whether v (assumed to be a -// 0#element, in the ABNF extension described in RFC 7230 section 7) -// contains token amongst its comma-separated tokens, ASCII -// case-insensitively. -func headerValueContainsToken(v string, token string) bool { - v = trimOWS(v) - if comma := strings.IndexByte(v, ','); comma != -1 { - return tokenEqual(trimOWS(v[:comma]), token) || headerValueContainsToken(v[comma+1:], token) - } - return tokenEqual(v, token) -} - -// lowerASCII returns the ASCII lowercase version of b. -func lowerASCII(b byte) byte { - if 'A' <= b && b <= 'Z' { - return b + ('a' - 'A') - } - return b -} - -// tokenEqual reports whether t1 and t2 are equal, ASCII case-insensitively. -func tokenEqual(t1, t2 string) bool { - if len(t1) != len(t2) { - return false - } - for i, b := range t1 { - if b >= utf8.RuneSelf { - // No UTF-8 or non-ASCII allowed in tokens. - return false - } - if lowerASCII(byte(b)) != lowerASCII(t2[i]) { - return false - } - } - return true -} - -// isLWS reports whether b is linear white space, according -// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 -// LWS = [CRLF] 1*( SP | HT ) -func isLWS(b byte) bool { return b == ' ' || b == '\t' } - -// isCTL reports whether b is a control byte, according -// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 -// CTL = <any US-ASCII control character -// (octets 0 - 31) and DEL (127)> -func isCTL(b byte) bool { - const del = 0x7f // a CTL - return b < ' ' || b == del -} diff --git a/libgo/go/net/http/lex_test.go b/libgo/go/net/http/lex_test.go deleted file mode 100644 index 986fda1..0000000 --- a/libgo/go/net/http/lex_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package http - -import ( - "testing" -) - -func isChar(c rune) bool { return c <= 127 } - -func isCtl(c rune) bool { return c <= 31 || c == 127 } - -func isSeparator(c rune) bool { - switch c { - case '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}', ' ', '\t': - return true - } - return false -} - -func TestIsToken(t *testing.T) { - for i := 0; i <= 130; i++ { - r := rune(i) - expected := isChar(r) && !isCtl(r) && !isSeparator(r) - if isToken(r) != expected { - t.Errorf("isToken(0x%x) = %v", r, !expected) - } - } -} - -func TestHeaderValuesContainsToken(t *testing.T) { - tests := []struct { - vals []string - token string - want bool - }{ - { - vals: []string{"foo"}, - token: "foo", - want: true, - }, - { - vals: []string{"bar", "foo"}, - token: "foo", - want: true, - }, - { - vals: []string{"foo"}, - token: "FOO", - want: true, - }, - { - vals: []string{"foo"}, - token: "bar", - want: false, - }, - { - vals: []string{" foo "}, - token: "FOO", - want: true, - }, - { - vals: []string{"foo,bar"}, - token: "FOO", - want: true, - }, - { - vals: []string{"bar,foo,bar"}, - token: "FOO", - want: true, - }, - { - vals: []string{"bar , foo"}, - token: "FOO", - want: true, - }, - { - vals: []string{"foo ,bar "}, - token: "FOO", - want: true, - }, - { - vals: []string{"bar, foo ,bar"}, - token: "FOO", - want: true, - }, - { - vals: []string{"bar , foo"}, - token: "FOO", - want: true, - }, - } - for _, tt := range tests { - got := headerValuesContainsToken(tt.vals, tt.token) - if got != tt.want { - t.Errorf("headerValuesContainsToken(%q, %q) = %v; want %v", tt.vals, tt.token, got, tt.want) - } - } -} diff --git a/libgo/go/net/http/main_test.go b/libgo/go/net/http/main_test.go index 299cd7b..aea6e12 100644 --- a/libgo/go/net/http/main_test.go +++ b/libgo/go/net/http/main_test.go @@ -5,7 +5,6 @@ package http_test import ( - "flag" "fmt" "net/http" "os" @@ -16,8 +15,6 @@ import ( "time" ) -var flaky = flag.Bool("flaky", false, "run known-flaky tests too") - func TestMain(m *testing.M) { v := m.Run() if v == 0 && goroutineLeaked() { @@ -91,12 +88,6 @@ func setParallel(t *testing.T) { } } -func setFlaky(t *testing.T, issue int) { - if !*flaky { - t.Skipf("skipping known flaky test; see golang.org/issue/%d", issue) - } -} - func afterTest(t testing.TB) { http.DefaultTransport.(*http.Transport).CloseIdleConnections() if testing.Short() { @@ -129,3 +120,17 @@ func afterTest(t testing.TB) { } t.Errorf("Test appears to have leaked %s:\n%s", bad, stacks) } + +// waitCondition reports whether fn eventually returned true, +// checking immediately and then every checkEvery amount, +// until waitFor has elapsed, at which point it returns false. +func waitCondition(waitFor, checkEvery time.Duration, fn func() bool) bool { + deadline := time.Now().Add(waitFor) + for time.Now().Before(deadline) { + if fn() { + return true + } + time.Sleep(checkEvery) + } + return false +} diff --git a/libgo/go/net/http/method.go b/libgo/go/net/http/method.go index b74f960..6f46155 100644 --- a/libgo/go/net/http/method.go +++ b/libgo/go/net/http/method.go @@ -12,7 +12,7 @@ const ( MethodHead = "HEAD" MethodPost = "POST" MethodPut = "PUT" - MethodPatch = "PATCH" // RFC 5741 + MethodPatch = "PATCH" // RFC 5789 MethodDelete = "DELETE" MethodConnect = "CONNECT" MethodOptions = "OPTIONS" diff --git a/libgo/go/net/http/pprof/pprof.go b/libgo/go/net/http/pprof/pprof.go index 7262c6c..05d0890 100644 --- a/libgo/go/net/http/pprof/pprof.go +++ b/libgo/go/net/http/pprof/pprof.go @@ -1,11 +1,9 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package pprof serves via its HTTP server runtime profiling data // in the format expected by the pprof visualization tool. -// For more information about pprof, see -// http://code.google.com/p/google-perftools/. // // The package is typically only imported for the side effect of // registering its HTTP handlers. @@ -15,7 +13,7 @@ // import _ "net/http/pprof" // // If your application is not already running an http server, you -// need to start one. Add "net/http" and "log" to your imports and +// need to start one. Add "net/http" and "log" to your imports and // the following code to your main function: // // go func() { @@ -30,7 +28,8 @@ // // go tool pprof http://localhost:6060/debug/pprof/profile // -// Or to look at the goroutine blocking profile: +// Or to look at the goroutine blocking profile, after calling +// runtime.SetBlockProfileRate in your program: // // go tool pprof http://localhost:6060/debug/pprof/block // @@ -118,8 +117,8 @@ func Profile(w http.ResponseWriter, r *http.Request) { // Tracing lasts for duration specified in seconds GET parameter, or for 1 second if not specified. // The package initialization registers it as /debug/pprof/trace. func Trace(w http.ResponseWriter, r *http.Request) { - sec, _ := strconv.ParseInt(r.FormValue("seconds"), 10, 64) - if sec == 0 { + sec, err := strconv.ParseFloat(r.FormValue("seconds"), 64) + if sec <= 0 || err != nil { sec = 1 } @@ -127,18 +126,16 @@ func Trace(w http.ResponseWriter, r *http.Request) { // because if it does it starts writing. w.Header().Set("Content-Type", "application/octet-stream") w.Write([]byte("tracing not yet supported with gccgo")) - /* - if err := trace.Start(w); err != nil { - // trace.Start failed, so no writes yet. - // Can change header back to text content and send error code. - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "Could not enable tracing: %s\n", err) - return - } - sleep(w, time.Duration(sec)*time.Second) - trace.Stop() - */ + // if err := trace.Start(w); err != nil { + // // trace.Start failed, so no writes yet. + // // Can change header back to text content and send error code. + // w.Header().Set("Content-Type", "text/plain; charset=utf-8") + // w.WriteHeader(http.StatusInternalServerError) + // fmt.Fprintf(w, "Could not enable tracing: %s\n", err) + // return + // } + // sleep(w, time.Duration(sec*float64(time.Second))) + // trace.Stop() } // Symbol looks up the program counters listed in the request, @@ -148,11 +145,11 @@ func Symbol(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") // We have to read the whole POST body before - // writing any output. Buffer the output here. + // writing any output. Buffer the output here. var buf bytes.Buffer // We don't know how many symbols we have, but we - // do have symbol information. Pprof only cares whether + // do have symbol information. Pprof only cares whether // this number is 0 (no symbols available) or > 0. fmt.Fprintf(&buf, "num_symbols: 1\n") diff --git a/libgo/go/net/http/readrequest_test.go b/libgo/go/net/http/readrequest_test.go index 60e2be4..4bf646b 100644 --- a/libgo/go/net/http/readrequest_test.go +++ b/libgo/go/net/http/readrequest_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -380,6 +380,27 @@ var reqTests = []reqTest{ noTrailer, noError, }, + + // http2 client preface: + { + "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n", + &Request{ + Method: "PRI", + URL: &url.URL{ + Path: "*", + }, + Header: Header{}, + Proto: "HTTP/2.0", + ProtoMajor: 2, + ProtoMinor: 0, + RequestURI: "*", + ContentLength: -1, + Close: true, + }, + noBody, + noTrailer, + noError, + }, } func TestReadRequest(t *testing.T) { diff --git a/libgo/go/net/http/request.go b/libgo/go/net/http/request.go index 8cdab02..dc55592 100644 --- a/libgo/go/net/http/request.go +++ b/libgo/go/net/http/request.go @@ -9,6 +9,7 @@ package http import ( "bufio" "bytes" + "context" "crypto/tls" "encoding/base64" "errors" @@ -17,6 +18,7 @@ import ( "io/ioutil" "mime" "mime/multipart" + "net/http/httptrace" "net/textproto" "net/url" "strconv" @@ -247,7 +249,52 @@ type Request struct { // RoundTripper may support Cancel. // // For server requests, this field is not applicable. + // + // Deprecated: Use the Context and WithContext methods + // instead. If a Request's Cancel field and context are both + // set, it is undefined whether Cancel is respected. Cancel <-chan struct{} + + // Response is the redirect response which caused this request + // to be created. This field is only populated during client + // redirects. + Response *Response + + // ctx is either the client or server context. It should only + // be modified via copying the whole Request using WithContext. + // It is unexported to prevent people from using Context wrong + // and mutating the contexts held by callers of the same request. + ctx context.Context +} + +// Context returns the request's context. To change the context, use +// WithContext. +// +// The returned context is always non-nil; it defaults to the +// background context. +// +// For outgoing client requests, the context controls cancelation. +// +// For incoming server requests, the context is canceled when the +// ServeHTTP method returns. For its associated values, see +// ServerContextKey and LocalAddrContextKey. +func (r *Request) Context() context.Context { + if r.ctx != nil { + return r.ctx + } + return context.Background() +} + +// WithContext returns a shallow copy of r with its context changed +// to ctx. The provided ctx must be non-nil. +func (r *Request) WithContext(ctx context.Context) *Request { + if ctx == nil { + panic("nil context") + } + r2 := new(Request) + *r2 = *r + r2.ctx = ctx + return r2 } // ProtoAtLeast reports whether the HTTP protocol used @@ -279,8 +326,8 @@ func (r *Request) Cookie(name string) (*Cookie, error) { return nil, ErrNoCookie } -// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, -// AddCookie does not attach more than one Cookie header field. That +// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, +// AddCookie does not attach more than one Cookie header field. That // means all cookies, if any, are written into the same line, // separated by semicolon. func (r *Request) AddCookie(c *Cookie) { @@ -343,6 +390,12 @@ func (r *Request) multipartReader() (*multipart.Reader, error) { return multipart.NewReader(r.Body, boundary), nil } +// isH2Upgrade reports whether r represents the http2 "client preface" +// magic string. +func (r *Request) isH2Upgrade() bool { + return r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" +} + // Return value if nonempty, def otherwise. func valueOrDefault(value, def string) string { if value != "" { @@ -375,7 +428,7 @@ func (r *Request) Write(w io.Writer) error { } // WriteProxy is like Write but writes the request in the form -// expected by an HTTP proxy. In particular, WriteProxy writes the +// expected by an HTTP proxy. In particular, WriteProxy writes the // initial Request-URI line of the request with an absolute URI, per // section 5.1.2 of RFC 2616, including the scheme and host. // In either case, WriteProxy also writes a Host header, using @@ -390,7 +443,16 @@ var errMissingHost = errors.New("http: Request.Write on Request with no Host or // extraHeaders may be nil // waitForContinue may be nil -func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitForContinue func() bool) error { +func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitForContinue func() bool) (err error) { + trace := httptrace.ContextClientTrace(req.Context()) + if trace != nil && trace.WroteRequest != nil { + defer func() { + trace.WroteRequest(httptrace.WroteRequestInfo{ + Err: err, + }) + }() + } + // Find the target host. Prefer the Host: header, but if that // is not given, use the host from the request URL. // @@ -427,7 +489,7 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai w = bw } - _, err := fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri) + _, err = fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri) if err != nil { return err } @@ -478,6 +540,10 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai return err } + if trace != nil && trace.WroteHeaders != nil { + trace.WroteHeaders() + } + // Flush and wait for 100-continue if expected. if waitForContinue != nil { if bw, ok := w.(*bufio.Writer); ok { @@ -486,7 +552,9 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai return err } } - + if trace != nil && trace.Wait100Continue != nil { + trace.Wait100Continue() + } if !waitForContinue() { req.closeBody() return nil @@ -521,7 +589,7 @@ func cleanHost(in string) string { return in } -// removeZone removes IPv6 zone identifer from host. +// removeZone removes IPv6 zone identifier from host. // E.g., "[fe80::1%en0]:8080" to "[fe80::1]:8080" func removeZone(host string) string { if !strings.HasPrefix(host, "[") { @@ -613,6 +681,8 @@ func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { if !ok && body != nil { rc = ioutil.NopCloser(body) } + // The host's colon:port should be normalized. See Issue 14836. + u.Host = removeEmptyPort(u.Host) req := &Request{ Method: method, URL: u, @@ -704,7 +774,9 @@ func putTextprotoReader(r *textproto.Reader) { } // ReadRequest reads and parses an incoming request from b. -func ReadRequest(b *bufio.Reader) (req *Request, err error) { return readRequest(b, deleteHostHeader) } +func ReadRequest(b *bufio.Reader) (*Request, error) { + return readRequest(b, deleteHostHeader) +} // Constants for readRequest's deleteHostHeader parameter. const ( @@ -768,13 +840,13 @@ func readRequest(b *bufio.Reader, deleteHostHeader bool) (req *Request, err erro } req.Header = Header(mimeHeader) - // RFC2616: Must treat + // RFC 2616: Must treat // GET /index.html HTTP/1.1 // Host: www.google.com // and // GET http://www.google.com/index.html HTTP/1.1 // Host: doesntmatter - // the same. In the second case, any Host line is ignored. + // the same. In the second case, any Host line is ignored. req.Host = req.URL.Host if req.Host == "" { req.Host = req.Header.get("Host") @@ -792,6 +864,16 @@ func readRequest(b *bufio.Reader, deleteHostHeader bool) (req *Request, err erro return nil, err } + if req.isH2Upgrade() { + // Because it's neither chunked, nor declared: + req.ContentLength = -1 + + // We want to give handlers a chance to hijack the + // connection, but we need to prevent the Server from + // dealing with the connection further if it's not + // hijacked. Set Close to ensure that: + req.Close = true + } return req, nil } @@ -808,57 +890,56 @@ func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { } type maxBytesReader struct { - w ResponseWriter - r io.ReadCloser // underlying reader - n int64 // max bytes remaining - stopped bool - sawEOF bool + w ResponseWriter + r io.ReadCloser // underlying reader + n int64 // max bytes remaining + err error // sticky error } func (l *maxBytesReader) tooLarge() (n int, err error) { - if !l.stopped { - l.stopped = true - if res, ok := l.w.(*response); ok { - res.requestTooLarge() - } - } - return 0, errors.New("http: request body too large") + l.err = errors.New("http: request body too large") + return 0, l.err } func (l *maxBytesReader) Read(p []byte) (n int, err error) { - toRead := l.n - if l.n == 0 { - if l.sawEOF { - return l.tooLarge() - } - // The underlying io.Reader may not return (0, io.EOF) - // at EOF if the requested size is 0, so read 1 byte - // instead. The io.Reader docs are a bit ambiguous - // about the return value of Read when 0 bytes are - // requested, and {bytes,strings}.Reader gets it wrong - // too (it returns (0, nil) even at EOF). - toRead = 1 + if l.err != nil { + return 0, l.err } - if int64(len(p)) > toRead { - p = p[:toRead] + if len(p) == 0 { + return 0, nil + } + // If they asked for a 32KB byte read but only 5 bytes are + // remaining, no need to read 32KB. 6 bytes will answer the + // question of the whether we hit the limit or go past it. + if int64(len(p)) > l.n+1 { + p = p[:l.n+1] } n, err = l.r.Read(p) - if err == io.EOF { - l.sawEOF = true - } - if l.n == 0 { - // If we had zero bytes to read remaining (but hadn't seen EOF) - // and we get a byte here, that means we went over our limit. - if n > 0 { - return l.tooLarge() - } - return 0, err + + if int64(n) <= l.n { + l.n -= int64(n) + l.err = err + return n, err } - l.n -= int64(n) - if l.n < 0 { - l.n = 0 + + n = int(l.n) + l.n = 0 + + // The server code and client code both use + // maxBytesReader. This "requestTooLarge" check is + // only used by the server code. To prevent binaries + // which only using the HTTP Client code (such as + // cmd/go) from also linking in the HTTP server, don't + // use a static type assertion to the server + // "*response" type. Check this interface instead: + type requestTooLarger interface { + requestTooLarge() } - return + if res, ok := l.w.(requestTooLarger); ok { + res.requestTooLarge() + } + l.err = errors.New("http: request body too large") + return n, l.err } func (l *maxBytesReader) Close() error { @@ -995,9 +1076,16 @@ func (r *Request) ParseMultipartForm(maxMemory int64) error { if err != nil { return err } + + if r.PostForm == nil { + r.PostForm = make(url.Values) + } for k, v := range f.Value { r.Form[k] = append(r.Form[k], v...) + // r.PostForm should also be populated. See Issue 9305. + r.PostForm[k] = append(r.PostForm[k], v...) } + r.MultipartForm = f return nil @@ -1086,92 +1174,3 @@ func (r *Request) isReplayable() bool { } return false } - -func validHostHeader(h string) bool { - // The latests spec is actually this: - // - // http://tools.ietf.org/html/rfc7230#section-5.4 - // Host = uri-host [ ":" port ] - // - // Where uri-host is: - // http://tools.ietf.org/html/rfc3986#section-3.2.2 - // - // But we're going to be much more lenient for now and just - // search for any byte that's not a valid byte in any of those - // expressions. - for i := 0; i < len(h); i++ { - if !validHostByte[h[i]] { - return false - } - } - return true -} - -// See the validHostHeader comment. -var validHostByte = [256]bool{ - '0': true, '1': true, '2': true, '3': true, '4': true, '5': true, '6': true, '7': true, - '8': true, '9': true, - - 'a': true, 'b': true, 'c': true, 'd': true, 'e': true, 'f': true, 'g': true, 'h': true, - 'i': true, 'j': true, 'k': true, 'l': true, 'm': true, 'n': true, 'o': true, 'p': true, - 'q': true, 'r': true, 's': true, 't': true, 'u': true, 'v': true, 'w': true, 'x': true, - 'y': true, 'z': true, - - 'A': true, 'B': true, 'C': true, 'D': true, 'E': true, 'F': true, 'G': true, 'H': true, - 'I': true, 'J': true, 'K': true, 'L': true, 'M': true, 'N': true, 'O': true, 'P': true, - 'Q': true, 'R': true, 'S': true, 'T': true, 'U': true, 'V': true, 'W': true, 'X': true, - 'Y': true, 'Z': true, - - '!': true, // sub-delims - '$': true, // sub-delims - '%': true, // pct-encoded (and used in IPv6 zones) - '&': true, // sub-delims - '(': true, // sub-delims - ')': true, // sub-delims - '*': true, // sub-delims - '+': true, // sub-delims - ',': true, // sub-delims - '-': true, // unreserved - '.': true, // unreserved - ':': true, // IPv6address + Host expression's optional port - ';': true, // sub-delims - '=': true, // sub-delims - '[': true, - '\'': true, // sub-delims - ']': true, - '_': true, // unreserved - '~': true, // unreserved -} - -func validHeaderName(v string) bool { - if len(v) == 0 { - return false - } - return strings.IndexFunc(v, isNotToken) == -1 -} - -// validHeaderValue reports whether v is a valid "field-value" according to -// http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 : -// -// message-header = field-name ":" [ field-value ] -// field-value = *( field-content | LWS ) -// field-content = <the OCTETs making up the field-value -// and consisting of either *TEXT or combinations -// of token, separators, and quoted-string> -// -// http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 : -// -// TEXT = <any OCTET except CTLs, -// but including LWS> -// LWS = [CRLF] 1*( SP | HT ) -// CTL = <any US-ASCII control character -// (octets 0 - 31) and DEL (127)> -func validHeaderValue(v string) bool { - for i := 0; i < len(v); i++ { - b := v[i] - if isCTL(b) && !isLWS(b) { - return false - } - } - return true -} diff --git a/libgo/go/net/http/request_test.go b/libgo/go/net/http/request_test.go index 0ecdf85..a4c88c0 100644 --- a/libgo/go/net/http/request_test.go +++ b/libgo/go/net/http/request_test.go @@ -99,7 +99,7 @@ type parseContentTypeTest struct { var parseContentTypeTests = []parseContentTypeTest{ {false, stringMap{"Content-Type": {"text/plain"}}}, - // Empty content type is legal - shoult be treated as + // Empty content type is legal - should be treated as // application/octet-stream (RFC 2616, section 7.2.1) {false, stringMap{}}, {true, stringMap{"Content-Type": {"text/plain; boundary="}}}, @@ -158,6 +158,68 @@ func TestMultipartReader(t *testing.T) { } } +// Issue 9305: ParseMultipartForm should populate PostForm too +func TestParseMultipartFormPopulatesPostForm(t *testing.T) { + postData := + `--xxx +Content-Disposition: form-data; name="field1" + +value1 +--xxx +Content-Disposition: form-data; name="field2" + +value2 +--xxx +Content-Disposition: form-data; name="file"; filename="file" +Content-Type: application/octet-stream +Content-Transfer-Encoding: binary + +binary data +--xxx-- +` + req := &Request{ + Method: "POST", + Header: Header{"Content-Type": {`multipart/form-data; boundary=xxx`}}, + Body: ioutil.NopCloser(strings.NewReader(postData)), + } + + initialFormItems := map[string]string{ + "language": "Go", + "name": "gopher", + "skill": "go-ing", + "field2": "initial-value2", + } + + req.Form = make(url.Values) + for k, v := range initialFormItems { + req.Form.Add(k, v) + } + + err := req.ParseMultipartForm(10000) + if err != nil { + t.Fatalf("unexpected multipart error %v", err) + } + + wantForm := url.Values{ + "language": []string{"Go"}, + "name": []string{"gopher"}, + "skill": []string{"go-ing"}, + "field1": []string{"value1"}, + "field2": []string{"initial-value2", "value2"}, + } + if !reflect.DeepEqual(req.Form, wantForm) { + t.Fatalf("req.Form = %v, want %v", req.Form, wantForm) + } + + wantPostForm := url.Values{ + "field1": []string{"value1"}, + "field2": []string{"value2"}, + } + if !reflect.DeepEqual(req.PostForm, wantPostForm) { + t.Fatalf("req.PostForm = %v, want %v", req.PostForm, wantPostForm) + } +} + func TestParseMultipartForm(t *testing.T) { req := &Request{ Method: "POST", @@ -336,11 +398,13 @@ var newRequestHostTests = []struct { {"http://192.168.0.1/", "192.168.0.1"}, {"http://192.168.0.1:8080/", "192.168.0.1:8080"}, + {"http://192.168.0.1:/", "192.168.0.1"}, {"http://[fe80::1]/", "[fe80::1]"}, {"http://[fe80::1]:8080/", "[fe80::1]:8080"}, {"http://[fe80::1%25en0]/", "[fe80::1%en0]"}, {"http://[fe80::1%25en0]:8080/", "[fe80::1%en0]:8080"}, + {"http://[fe80::1%25en0]:/", "[fe80::1%en0]"}, } func TestNewRequestHost(t *testing.T) { @@ -615,6 +679,46 @@ func TestIssue10884_MaxBytesEOF(t *testing.T) { } } +// Issue 14981: MaxBytesReader's return error wasn't sticky. It +// doesn't technically need to be, but people expected it to be. +func TestMaxBytesReaderStickyError(t *testing.T) { + isSticky := func(r io.Reader) error { + var log bytes.Buffer + buf := make([]byte, 1000) + var firstErr error + for { + n, err := r.Read(buf) + fmt.Fprintf(&log, "Read(%d) = %d, %v\n", len(buf), n, err) + if err == nil { + continue + } + if firstErr == nil { + firstErr = err + continue + } + if !reflect.DeepEqual(err, firstErr) { + return fmt.Errorf("non-sticky error. got log:\n%s", log.Bytes()) + } + t.Logf("Got log: %s", log.Bytes()) + return nil + } + } + tests := [...]struct { + readable int + limit int64 + }{ + 0: {99, 100}, + 1: {100, 100}, + 2: {101, 100}, + } + for i, tt := range tests { + rc := MaxBytesReader(nil, ioutil.NopCloser(bytes.NewReader(make([]byte, tt.readable))), tt.limit) + if err := isSticky(rc); err != nil { + t.Errorf("%d. error: %v", i, err) + } + } +} + func testMissingFile(t *testing.T, req *Request) { f, fh, err := req.FormFile("missing") if f != nil { diff --git a/libgo/go/net/http/requestwrite_test.go b/libgo/go/net/http/requestwrite_test.go index cfb95b0..2545f6f 100644 --- a/libgo/go/net/http/requestwrite_test.go +++ b/libgo/go/net/http/requestwrite_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -573,7 +573,7 @@ func TestRequestWriteClosesBody(t *testing.T) { "Transfer-Encoding: chunked\r\n\r\n" + // TODO: currently we don't buffer before chunking, so we get a // single "m" chunk before the other chunks, as this was the 1-byte - // read from our MultiReader where we stiched the Body back together + // read from our MultiReader where we stitched the Body back together // after sniffing whether the Body was 0 bytes or not. chunk("m") + chunk("y body") + @@ -604,7 +604,7 @@ func TestRequestWriteError(t *testing.T) { failAfter, writeCount := 0, 0 errFail := errors.New("fake write failure") - // w is the buffered io.Writer to write the request to. It + // w is the buffered io.Writer to write the request to. It // fails exactly once on its Nth Write call, as controlled by // failAfter. It also tracks the number of calls in // writeCount. diff --git a/libgo/go/net/http/response.go b/libgo/go/net/http/response.go index c424f61..5450d50 100644 --- a/libgo/go/net/http/response.go +++ b/libgo/go/net/http/response.go @@ -11,6 +11,7 @@ import ( "bytes" "crypto/tls" "errors" + "fmt" "io" "net/textproto" "net/url" @@ -33,7 +34,7 @@ type Response struct { ProtoMajor int // e.g. 1 ProtoMinor int // e.g. 0 - // Header maps header keys to values. If the response had multiple + // Header maps header keys to values. If the response had multiple // headers with the same key, they may be concatenated, with comma // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers // be semantically equivalent to a comma-delimited sequence.) Values @@ -57,8 +58,8 @@ type Response struct { // with a "chunked" Transfer-Encoding. Body io.ReadCloser - // ContentLength records the length of the associated content. The - // value -1 indicates that the length is unknown. Unless Request.Method + // ContentLength records the length of the associated content. The + // value -1 indicates that the length is unknown. Unless Request.Method // is "HEAD", values >= 0 indicate that the given number of bytes may // be read from Body. ContentLength int64 @@ -68,10 +69,19 @@ type Response struct { TransferEncoding []string // Close records whether the header directed that the connection be - // closed after reading Body. The value is advice for clients: neither + // closed after reading Body. The value is advice for clients: neither // ReadResponse nor Response.Write ever closes a connection. Close bool + // Uncompressed reports whether the response was sent compressed but + // was decompressed by the http package. When true, reading from + // Body yields the uncompressed content instead of the compressed + // content actually set from the server, ContentLength is set to -1, + // and the "Content-Length" and "Content-Encoding" fields are deleted + // from the responseHeader. To get the original response from + // the server, set Transport.DisableCompression to true. + Uncompressed bool + // Trailer maps trailer keys to values in the same // format as Header. // @@ -86,7 +96,7 @@ type Response struct { // any trailer values sent by the server. Trailer Header - // The Request that was sent to obtain this Response. + // Request is the request that was sent to obtain this Response. // Request's Body is nil (having already been consumed). // This is only populated for Client requests. Request *Request @@ -108,8 +118,8 @@ func (r *Response) Cookies() []*Cookie { var ErrNoLocation = errors.New("http: no Location header in response") // Location returns the URL of the response's "Location" header, -// if present. Relative redirects are resolved relative to -// the Response's Request. ErrNoLocation is returned if no +// if present. Relative redirects are resolved relative to +// the Response's Request. ErrNoLocation is returned if no // Location header is present. func (r *Response) Location() (*url.URL, error) { lv := r.Header.Get("Location") @@ -184,7 +194,7 @@ func ReadResponse(r *bufio.Reader, req *Request) (*Response, error) { return resp, nil } -// RFC2616: Should treat +// RFC 2616: Should treat // Pragma: no-cache // like // Cache-Control: no-cache @@ -203,7 +213,7 @@ func (r *Response) ProtoAtLeast(major, minor int) bool { r.ProtoMajor == major && r.ProtoMinor >= minor } -// Write writes r to w in the HTTP/1.n server response format, +// Write writes r to w in the HTTP/1.x server response format, // including the status line, headers, body, and optional trailer. // // This method consults the following fields of the response r: @@ -228,11 +238,13 @@ func (r *Response) Write(w io.Writer) error { if !ok { text = "status code " + strconv.Itoa(r.StatusCode) } + } else { + // Just to reduce stutter, if user set r.Status to "200 OK" and StatusCode to 200. + // Not important. + text = strings.TrimPrefix(text, strconv.Itoa(r.StatusCode)+" ") } - protoMajor, protoMinor := strconv.Itoa(r.ProtoMajor), strconv.Itoa(r.ProtoMinor) - statusCode := strconv.Itoa(r.StatusCode) + " " - text = strings.TrimPrefix(text, statusCode) - if _, err := io.WriteString(w, "HTTP/"+protoMajor+"."+protoMinor+" "+statusCode+text+"\r\n"); err != nil { + + if _, err := fmt.Fprintf(w, "HTTP/%d.%d %03d %s\r\n", r.ProtoMajor, r.ProtoMinor, r.StatusCode, text); err != nil { return err } @@ -265,7 +277,7 @@ func (r *Response) Write(w io.Writer) error { // content-length, the only way to do that is the old HTTP/1.0 // way, by noting the EOF with a connection close, so we need // to set Close. - if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !chunked(r1.TransferEncoding) { + if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !chunked(r1.TransferEncoding) && !r1.Uncompressed { r1.Close = true } diff --git a/libgo/go/net/http/response_test.go b/libgo/go/net/http/response_test.go index d8a5340..126da92 100644 --- a/libgo/go/net/http/response_test.go +++ b/libgo/go/net/http/response_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -10,6 +10,7 @@ import ( "compress/gzip" "crypto/rand" "fmt" + "go/ast" "io" "io/ioutil" "net/http/internal" @@ -505,6 +506,32 @@ some body`, "Body here\n", }, + + { + "HTTP/1.1 200 OK\r\n" + + "Content-Encoding: gzip\r\n" + + "Content-Length: 23\r\n" + + "Connection: keep-alive\r\n" + + "Keep-Alive: timeout=7200\r\n\r\n" + + "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00", + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), + Header: Header{ + "Content-Length": {"23"}, + "Content-Encoding": {"gzip"}, + "Connection": {"keep-alive"}, + "Keep-Alive": {"timeout=7200"}, + }, + Close: false, + ContentLength: 23, + }, + "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00", + }, } // tests successful calls to ReadResponse, and inspects the returned Response. @@ -656,10 +683,14 @@ func diff(t *testing.T, prefix string, have, want interface{}) { t.Errorf("%s: type mismatch %v want %v", prefix, hv.Type(), wv.Type()) } for i := 0; i < hv.NumField(); i++ { + name := hv.Type().Field(i).Name + if !ast.IsExported(name) { + continue + } hf := hv.Field(i).Interface() wf := wv.Field(i).Interface() if !reflect.DeepEqual(hf, wf) { - t.Errorf("%s: %s = %v want %v", prefix, hv.Type().Field(i).Name, hf, wf) + t.Errorf("%s: %s = %v want %v", prefix, name, hf, wf) } } } diff --git a/libgo/go/net/http/responsewrite_test.go b/libgo/go/net/http/responsewrite_test.go index 5b8d47a..90f6767 100644 --- a/libgo/go/net/http/responsewrite_test.go +++ b/libgo/go/net/http/responsewrite_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -222,6 +222,39 @@ func TestResponseWrite(t *testing.T) { }, "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nabcdef", }, + + // Status code under 100 should be zero-padded to + // three digits. Still bogus, but less bogus. (be + // consistent with generating three digits, since the + // Transport requires it) + { + Response{ + StatusCode: 7, + Status: "license to violate specs", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{}, + Body: nil, + }, + + "HTTP/1.0 007 license to violate specs\r\nContent-Length: 0\r\n\r\n", + }, + + // No stutter. + { + Response{ + StatusCode: 123, + Status: "123 Sesame Street", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{}, + Body: nil, + }, + + "HTTP/1.0 123 Sesame Street\r\nContent-Length: 0\r\n\r\n", + }, } for i := range respWriteTests { diff --git a/libgo/go/net/http/serve_test.go b/libgo/go/net/http/serve_test.go index 384b453..139ce3e 100644 --- a/libgo/go/net/http/serve_test.go +++ b/libgo/go/net/http/serve_test.go @@ -9,7 +9,10 @@ package http_test import ( "bufio" "bytes" + "compress/gzip" + "context" "crypto/tls" + "encoding/json" "errors" "fmt" "internal/testenv" @@ -617,7 +620,7 @@ func TestIdentityResponse(t *testing.T) { defer ts.Close() // Note: this relies on the assumption (which is true) that - // Get sends HTTP/1.1 or greater requests. Otherwise the + // Get sends HTTP/1.1 or greater requests. Otherwise the // server wouldn't have the choice to send back chunked // responses. for _, te := range []string{"", "identity"} { @@ -713,6 +716,31 @@ func testTCPConnectionCloses(t *testing.T, req string, h Handler) { } } +func testTCPConnectionStaysOpen(t *testing.T, req string, handler Handler) { + defer afterTest(t) + ts := httptest.NewServer(handler) + defer ts.Close() + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + br := bufio.NewReader(conn) + for i := 0; i < 2; i++ { + if _, err := io.WriteString(conn, req); err != nil { + t.Fatal(err) + } + res, err := ReadResponse(br, nil) + if err != nil { + t.Fatalf("res %d: %v", i+1, err) + } + if _, err := io.Copy(ioutil.Discard, res.Body); err != nil { + t.Fatalf("res %d body copy: %v", i+1, err) + } + res.Body.Close() + } +} + // TestServeHTTP10Close verifies that HTTP/1.0 requests won't be kept alive. func TestServeHTTP10Close(t *testing.T) { testTCPConnectionCloses(t, "GET / HTTP/1.0\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { @@ -741,6 +769,61 @@ func TestHandlersCanSetConnectionClose10(t *testing.T) { })) } +func TestHTTP2UpgradeClosesConnection(t *testing.T) { + testTCPConnectionCloses(t, "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + // Nothing. (if not hijacked, the server should close the connection + // afterwards) + })) +} + +func send204(w ResponseWriter, r *Request) { w.WriteHeader(204) } +func send304(w ResponseWriter, r *Request) { w.WriteHeader(304) } + +// Issue 15647: 204 responses can't have bodies, so HTTP/1.0 keep-alive conns should stay open. +func TestHTTP10KeepAlive204Response(t *testing.T) { + testTCPConnectionStaysOpen(t, "GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n", HandlerFunc(send204)) +} + +func TestHTTP11KeepAlive204Response(t *testing.T) { + testTCPConnectionStaysOpen(t, "GET / HTTP/1.1\r\nHost: foo\r\n\r\n", HandlerFunc(send204)) +} + +func TestHTTP10KeepAlive304Response(t *testing.T) { + testTCPConnectionStaysOpen(t, + "GET / HTTP/1.0\r\nConnection: keep-alive\r\nIf-Modified-Since: Mon, 02 Jan 2006 15:04:05 GMT\r\n\r\n", + HandlerFunc(send304)) +} + +// Issue 15703 +func TestKeepAliveFinalChunkWithEOF(t *testing.T) { + defer afterTest(t) + cst := newClientServerTest(t, false /* h1 */, HandlerFunc(func(w ResponseWriter, r *Request) { + w.(Flusher).Flush() // force chunked encoding + w.Write([]byte("{\"Addr\": \"" + r.RemoteAddr + "\"}")) + })) + defer cst.close() + type data struct { + Addr string + } + var addrs [2]data + for i := range addrs { + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + if err := json.NewDecoder(res.Body).Decode(&addrs[i]); err != nil { + t.Fatal(err) + } + if addrs[i].Addr == "" { + t.Fatal("no address") + } + res.Body.Close() + } + if addrs[0] != addrs[1] { + t.Fatalf("connection not reused") + } +} + func TestSetsRemoteAddr_h1(t *testing.T) { testSetsRemoteAddr(t, h1Mode) } func TestSetsRemoteAddr_h2(t *testing.T) { testSetsRemoteAddr(t, h2Mode) } @@ -984,7 +1067,7 @@ func TestTLSServer(t *testing.T) { defer ts.Close() // Connect an idle TCP connection to this server before we run - // our real tests. This idle connection used to block forever + // our real tests. This idle connection used to block forever // in the TLS handshake, preventing future connections from // being accepted. It may prevent future accidental blocking // in newConn. @@ -1024,11 +1107,44 @@ func TestTLSServer(t *testing.T) { }) } -func TestAutomaticHTTP2_Serve(t *testing.T) { +// Issue 15908 +func TestAutomaticHTTP2_Serve_NoTLSConfig(t *testing.T) { + testAutomaticHTTP2_Serve(t, nil, true) +} + +func TestAutomaticHTTP2_Serve_NonH2TLSConfig(t *testing.T) { + testAutomaticHTTP2_Serve(t, &tls.Config{}, false) +} + +func TestAutomaticHTTP2_Serve_H2TLSConfig(t *testing.T) { + testAutomaticHTTP2_Serve(t, &tls.Config{NextProtos: []string{"h2"}}, true) +} + +func testAutomaticHTTP2_Serve(t *testing.T, tlsConf *tls.Config, wantH2 bool) { defer afterTest(t) ln := newLocalListener(t) ln.Close() // immediately (not a defer!) var s Server + s.TLSConfig = tlsConf + if err := s.Serve(ln); err == nil { + t.Fatal("expected an error") + } + gotH2 := s.TLSNextProto["h2"] != nil + if gotH2 != wantH2 { + t.Errorf("http2 configured = %v; want %v", gotH2, wantH2) + } +} + +func TestAutomaticHTTP2_Serve_WithTLSConfig(t *testing.T) { + defer afterTest(t) + ln := newLocalListener(t) + ln.Close() // immediately (not a defer!) + var s Server + // Set the TLSConfig. In reality, this would be the + // *tls.Config given to tls.NewListener. + s.TLSConfig = &tls.Config{ + NextProtos: []string{"h2"}, + } if err := s.Serve(ln); err == nil { t.Fatal("expected an error") } @@ -1888,6 +2004,52 @@ func TestTimeoutHandlerRaceHeaderTimeout(t *testing.T) { } } +// Issue 14568. +func TestTimeoutHandlerStartTimerWhenServing(t *testing.T) { + if testing.Short() { + t.Skip("skipping sleeping test in -short mode") + } + defer afterTest(t) + var handler HandlerFunc = func(w ResponseWriter, _ *Request) { + w.WriteHeader(StatusNoContent) + } + timeout := 300 * time.Millisecond + ts := httptest.NewServer(TimeoutHandler(handler, timeout, "")) + defer ts.Close() + // Issue was caused by the timeout handler starting the timer when + // was created, not when the request. So wait for more than the timeout + // to ensure that's not the case. + time.Sleep(2 * timeout) + res, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != StatusNoContent { + t.Errorf("got res.StatusCode %d, want %v", res.StatusCode, StatusNoContent) + } +} + +// https://golang.org/issue/15948 +func TestTimeoutHandlerEmptyResponse(t *testing.T) { + defer afterTest(t) + var handler HandlerFunc = func(w ResponseWriter, _ *Request) { + // No response. + } + timeout := 300 * time.Millisecond + ts := httptest.NewServer(TimeoutHandler(handler, timeout, "")) + defer ts.Close() + + res, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != StatusOK { + t.Errorf("got res.StatusCode %d, want %v", res.StatusCode, StatusOK) + } +} + // Verifies we don't path.Clean() on the wrong parts in redirects. func TestRedirectMunging(t *testing.T) { req, _ := NewRequest("GET", "http://example.com/", nil) @@ -2027,7 +2189,7 @@ func TestHandlerPanicWithHijack(t *testing.T) { func testHandlerPanic(t *testing.T, withHijack, h2 bool, panicValue interface{}) { defer afterTest(t) // Unlike the other tests that set the log output to ioutil.Discard - // to quiet the output, this test uses a pipe. The pipe serves three + // to quiet the output, this test uses a pipe. The pipe serves three // purposes: // // 1) The log.Print from the http server (generated by the caught @@ -2060,7 +2222,7 @@ func testHandlerPanic(t *testing.T, withHijack, h2 bool, panicValue interface{}) defer cst.close() // Do a blocking read on the log output pipe so its logging - // doesn't bleed into the next test. But wait only 5 seconds + // doesn't bleed into the next test. But wait only 5 seconds // for it. done := make(chan bool, 1) go func() { @@ -2205,10 +2367,10 @@ func testRequestBodyLimit(t *testing.T, h2 bool) { nWritten := new(int64) req, _ := NewRequest("POST", cst.ts.URL, io.LimitReader(countReader{neverEnding('a'), nWritten}, limit*200)) - // Send the POST, but don't care it succeeds or not. The + // Send the POST, but don't care it succeeds or not. The // remote side is going to reply and then close the TCP // connection, and HTTP doesn't really define if that's - // allowed or not. Some HTTP clients will get the response + // allowed or not. Some HTTP clients will get the response // and some (like ours, currently) will complain that the // request write failed, without reading the response. // @@ -2650,7 +2812,7 @@ func TestOptions(t *testing.T) { } // Tests regarding the ordering of Write, WriteHeader, Header, and -// Flush calls. In Go 1.0, rw.WriteHeader immediately flushed the +// Flush calls. In Go 1.0, rw.WriteHeader immediately flushed the // (*response).header to the wire. In Go 1.1, the actual wire flush is // delayed, so we could maybe tack on a Content-Length and better // Content-Type after we see more (or all) of the output. To preserve @@ -3107,7 +3269,7 @@ func testTransportAndServerSharedBodyRace(t *testing.T, h2 bool) { const bodySize = 1 << 20 - // errorf is like t.Errorf, but also writes to println. When + // errorf is like t.Errorf, but also writes to println. When // this test fails, it hangs. This helps debugging and I've // added this enough times "temporarily". It now gets added // full time. @@ -3829,6 +3991,8 @@ func TestServerValidatesHostHeader(t *testing.T) { host string want int }{ + {"HTTP/0.9", "", 400}, + {"HTTP/1.1", "", 400}, {"HTTP/1.1", "Host: \r\n", 200}, {"HTTP/1.1", "Host: 1.2.3.4\r\n", 200}, @@ -3851,10 +4015,22 @@ func TestServerValidatesHostHeader(t *testing.T) { {"HTTP/1.0", "", 200}, {"HTTP/1.0", "Host: first\r\nHost: second\r\n", 400}, {"HTTP/1.0", "Host: \xff\r\n", 400}, + + // Make an exception for HTTP upgrade requests: + {"PRI * HTTP/2.0", "", 200}, + + // But not other HTTP/2 stuff: + {"PRI / HTTP/2.0", "", 400}, + {"GET / HTTP/2.0", "", 400}, + {"GET / HTTP/3.0", "", 400}, } for _, tt := range tests { conn := &testConn{closec: make(chan bool, 1)} - io.WriteString(&conn.readBuf, "GET / "+tt.proto+"\r\n"+tt.host+"\r\n") + methodTarget := "GET / " + if !strings.HasPrefix(tt.proto, "HTTP/") { + methodTarget = "" + } + io.WriteString(&conn.readBuf, methodTarget+tt.proto+"\r\n"+tt.host+"\r\n") ln := &oneConnListener{conn} go Serve(ln, HandlerFunc(func(ResponseWriter, *Request) {})) @@ -3870,6 +4046,45 @@ func TestServerValidatesHostHeader(t *testing.T) { } } +func TestServerHandlersCanHandleH2PRI(t *testing.T) { + const upgradeResponse = "upgrade here" + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + conn, br, err := w.(Hijacker).Hijack() + defer conn.Close() + if r.Method != "PRI" || r.RequestURI != "*" { + t.Errorf("Got method/target %q %q; want PRI *", r.Method, r.RequestURI) + return + } + if !r.Close { + t.Errorf("Request.Close = true; want false") + } + const want = "SM\r\n\r\n" + buf := make([]byte, len(want)) + n, err := io.ReadFull(br, buf) + if err != nil || string(buf[:n]) != want { + t.Errorf("Read = %v, %v (%q), want %q", n, err, buf[:n], want) + return + } + io.WriteString(conn, upgradeResponse) + })) + defer ts.Close() + + c, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer c.Close() + io.WriteString(c, "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") + slurp, err := ioutil.ReadAll(c) + if err != nil { + t.Fatal(err) + } + if string(slurp) != upgradeResponse { + t.Errorf("Handler response = %q; want %q", slurp, upgradeResponse) + } +} + // Test that we validate the valid bytes in HTTP/1 headers. // Issue 11207. func TestServerValidatesHeaders(t *testing.T) { @@ -3910,6 +4125,140 @@ func TestServerValidatesHeaders(t *testing.T) { } } +func TestServerRequestContextCancel_ServeHTTPDone_h1(t *testing.T) { + testServerRequestContextCancel_ServeHTTPDone(t, h1Mode) +} +func TestServerRequestContextCancel_ServeHTTPDone_h2(t *testing.T) { + testServerRequestContextCancel_ServeHTTPDone(t, h2Mode) +} +func testServerRequestContextCancel_ServeHTTPDone(t *testing.T, h2 bool) { + defer afterTest(t) + ctxc := make(chan context.Context, 1) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + ctx := r.Context() + select { + case <-ctx.Done(): + t.Error("should not be Done in ServeHTTP") + default: + } + ctxc <- ctx + })) + defer cst.close() + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + ctx := <-ctxc + select { + case <-ctx.Done(): + default: + t.Error("context should be done after ServeHTTP completes") + } +} + +func TestServerRequestContextCancel_ConnClose(t *testing.T) { + // Currently the context is not canceled when the connection + // is closed because we're not reading from the connection + // until after ServeHTTP for the previous handler is done. + // Until the server code is modified to always be in a read + // (Issue 15224), this test doesn't work yet. + t.Skip("TODO(bradfitz): this test doesn't yet work; golang.org/issue/15224") + defer afterTest(t) + inHandler := make(chan struct{}) + handlerDone := make(chan struct{}) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + close(inHandler) + select { + case <-r.Context().Done(): + case <-time.After(3 * time.Second): + t.Errorf("timeout waiting for context to be done") + } + close(handlerDone) + })) + defer ts.Close() + c, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer c.Close() + io.WriteString(c, "GET / HTTP/1.1\r\nHost: foo\r\n\r\n") + select { + case <-inHandler: + case <-time.After(3 * time.Second): + t.Fatalf("timeout waiting to see ServeHTTP get called") + } + c.Close() // this should trigger the context being done + + select { + case <-handlerDone: + case <-time.After(3 * time.Second): + t.Fatalf("timeout waiting to see ServeHTTP exit") + } +} + +func TestServerContext_ServerContextKey_h1(t *testing.T) { + testServerContext_ServerContextKey(t, h1Mode) +} +func TestServerContext_ServerContextKey_h2(t *testing.T) { + testServerContext_ServerContextKey(t, h2Mode) +} +func testServerContext_ServerContextKey(t *testing.T, h2 bool) { + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + ctx := r.Context() + got := ctx.Value(ServerContextKey) + if _, ok := got.(*Server); !ok { + t.Errorf("context value = %T; want *http.Server", got) + } + + got = ctx.Value(LocalAddrContextKey) + if addr, ok := got.(net.Addr); !ok { + t.Errorf("local addr value = %T; want net.Addr", got) + } else if fmt.Sprint(addr) != r.Host { + t.Errorf("local addr = %v; want %v", addr, r.Host) + } + })) + defer cst.close() + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() +} + +// https://golang.org/issue/15960 +func TestHandlerSetTransferEncodingChunked(t *testing.T) { + defer afterTest(t) + ht := newHandlerTest(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Transfer-Encoding", "chunked") + w.Write([]byte("hello")) + })) + resp := ht.rawResponse("GET / HTTP/1.1\nHost: foo") + const hdr = "Transfer-Encoding: chunked" + if n := strings.Count(resp, hdr); n != 1 { + t.Errorf("want 1 occurrence of %q in response, got %v\nresponse: %v", hdr, n, resp) + } +} + +// https://golang.org/issue/16063 +func TestHandlerSetTransferEncodingGzip(t *testing.T) { + defer afterTest(t) + ht := newHandlerTest(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Transfer-Encoding", "gzip") + gz := gzip.NewWriter(w) + gz.Write([]byte("hello")) + gz.Close() + })) + resp := ht.rawResponse("GET / HTTP/1.1\nHost: foo") + for _, v := range []string{"gzip", "chunked"} { + hdr := "Transfer-Encoding: " + v + if n := strings.Count(resp, hdr); n != 1 { + t.Errorf("want 1 occurrence of %q in response, got %v\nresponse: %v", hdr, n, resp) + } + } +} + func BenchmarkClientServer(b *testing.B) { b.ReportAllocs() b.StopTimer() @@ -4100,7 +4449,7 @@ func BenchmarkClient(b *testing.B) { // Wait for the server process to respond. url := "http://localhost:" + port + "/" for i := 0; i < 100; i++ { - time.Sleep(50 * time.Millisecond) + time.Sleep(100 * time.Millisecond) if _, err := getNoBody(url); err == nil { break } @@ -4121,7 +4470,7 @@ func BenchmarkClient(b *testing.B) { if err != nil { b.Fatalf("ReadAll: %v", err) } - if bytes.Compare(body, data) != 0 { + if !bytes.Equal(body, data) { b.Fatalf("Got body: %q", body) } } diff --git a/libgo/go/net/http/server.go b/libgo/go/net/http/server.go index 5e3b608..7b2b4b2 100644 --- a/libgo/go/net/http/server.go +++ b/libgo/go/net/http/server.go @@ -2,13 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// HTTP server. See RFC 2616. +// HTTP server. See RFC 2616. package http import ( "bufio" "bytes" + "context" "crypto/tls" "errors" "fmt" @@ -26,14 +27,30 @@ import ( "sync" "sync/atomic" "time" + + "golang_org/x/net/lex/httplex" ) -// Errors introduced by the HTTP server. +// Errors used by the HTTP server. var ( - ErrWriteAfterFlush = errors.New("Conn.Write called after Flush") - ErrBodyNotAllowed = errors.New("http: request method or response status code does not allow body") - ErrHijacked = errors.New("Conn has been hijacked") - ErrContentLength = errors.New("Conn.Write wrote more than the declared Content-Length") + // ErrBodyNotAllowed is returned by ResponseWriter.Write calls + // when the HTTP method or response code does not permit a + // body. + ErrBodyNotAllowed = errors.New("http: request method or response status code does not allow body") + + // ErrHijacked is returned by ResponseWriter.Write calls when + // the underlying connection has been hijacked using the + // Hijacker interfaced. + ErrHijacked = errors.New("http: connection has been hijacked") + + // ErrContentLength is returned by ResponseWriter.Write calls + // when a Handler set a Content-Length response header with a + // declared size and then attempted to write more bytes than + // declared. + ErrContentLength = errors.New("http: wrote more than the declared Content-Length") + + // Deprecated: ErrWriteAfterFlush is no longer used. + ErrWriteAfterFlush = errors.New("unused") ) // A Handler responds to an HTTP request. @@ -50,6 +67,9 @@ var ( // ResponseWriter. Cautious handlers should read the Request.Body // first, and then reply. // +// Except for reading the body, handlers should not modify the +// provided Request. +// // If ServeHTTP panics, the server (the caller of ServeHTTP) assumes // that the effect of the panic was isolated to the active request. // It recovers the panic, logs a stack trace to the server error log, @@ -73,10 +93,24 @@ type ResponseWriter interface { Header() Header // Write writes the data to the connection as part of an HTTP reply. - // If WriteHeader has not yet been called, Write calls WriteHeader(http.StatusOK) - // before writing the data. If the Header does not contain a - // Content-Type line, Write adds a Content-Type set to the result of passing - // the initial 512 bytes of written data to DetectContentType. + // + // If WriteHeader has not yet been called, Write calls + // WriteHeader(http.StatusOK) before writing the data. If the Header + // does not contain a Content-Type line, Write adds a Content-Type set + // to the result of passing the initial 512 bytes of written data to + // DetectContentType. + // + // Depending on the HTTP protocol version and the client, calling + // Write or WriteHeader may prevent future reads on the + // Request.Body. For HTTP/1.x requests, handlers should read any + // needed request body data before writing the response. Once the + // headers have been flushed (due to either an explicit Flusher.Flush + // call or writing enough data to trigger a flush), the request body + // may be unavailable. For HTTP/2 requests, the Go HTTP server permits + // handlers to continue to read the request body while concurrently + // writing the response. However, such behavior may not be supported + // by all HTTP/2 clients. Handlers should read before writing if + // possible to maximize compatibility. Write([]byte) (int, error) // WriteHeader sends an HTTP response header with status code. @@ -90,6 +124,10 @@ type ResponseWriter interface { // The Flusher interface is implemented by ResponseWriters that allow // an HTTP handler to flush buffered data to the client. // +// The default HTTP/1.x and HTTP/2 ResponseWriter implementations +// support Flusher, but ResponseWriter wrappers may not. Handlers +// should always test for this ability at runtime. +// // Note that even for ResponseWriters that support Flush, // if the client is connected through an HTTP proxy, // the buffered data may not reach the client until the response @@ -101,6 +139,11 @@ type Flusher interface { // The Hijacker interface is implemented by ResponseWriters that allow // an HTTP handler to take over the connection. +// +// The default ResponseWriter for HTTP/1.x connections supports +// Hijacker, but HTTP/2 connections intentionally do not. +// ResponseWriter wrappers may also not support Hijacker. Handlers +// should always test for this ability at runtime. type Hijacker interface { // Hijack lets the caller take over the connection. // After a call to Hijack(), the HTTP server library @@ -143,6 +186,20 @@ type CloseNotifier interface { CloseNotify() <-chan bool } +var ( + // ServerContextKey is a context key. It can be used in HTTP + // handlers with context.WithValue to access the server that + // started the handler. The associated value will be of + // type *Server. + ServerContextKey = &contextKey{"http-server"} + + // LocalAddrContextKey is a context key. It can be used in + // HTTP handlers with context.WithValue to access the address + // the local address the connection arrived on. + // The associated value will be of type net.Addr. + LocalAddrContextKey = &contextKey{"local-addr"} +) + // A conn represents the server side of an HTTP connection. type conn struct { // server is the server on which the connection arrived. @@ -306,11 +363,14 @@ func (cw *chunkWriter) close() { // A response represents the server side of an HTTP response. type response struct { - conn *conn - req *Request // request for this response - reqBody io.ReadCloser - wroteHeader bool // reply header has been (logically) written - wroteContinue bool // 100 Continue response was written + conn *conn + req *Request // request for this response + reqBody io.ReadCloser + cancelCtx context.CancelFunc // when ServeHTTP exits + wroteHeader bool // reply header has been (logically) written + wroteContinue bool // 100 Continue response was written + wants10KeepAlive bool // HTTP/1.0 w/ Connection "keep-alive" + wantsClose bool // HTTP request has Connection "close" w *bufio.Writer // buffers output in chunks to chunkWriter cw chunkWriter @@ -342,7 +402,7 @@ type response struct { requestBodyLimitHit bool // trailers are the headers to be sent after the handler - // finishes writing the body. This field is initialized from + // finishes writing the body. This field is initialized from // the Trailer response header when the response header is // written. trailers []string @@ -497,7 +557,7 @@ type connReader struct { } func (cr *connReader) setReadLimit(remain int64) { cr.remain = remain } -func (cr *connReader) setInfiniteReadLimit() { cr.remain = 1<<63 - 1 } +func (cr *connReader) setInfiniteReadLimit() { cr.remain = maxInt64 } func (cr *connReader) hitReadLimit() bool { return cr.remain <= 0 } func (cr *connReader) Read(p []byte) (n int, err error) { @@ -681,7 +741,7 @@ func appendTime(b []byte, t time.Time) []byte { var errTooLarge = errors.New("http: request too large") // Read next request from connection. -func (c *conn) readRequest() (w *response, err error) { +func (c *conn) readRequest(ctx context.Context) (w *response, err error) { if c.hijacked() { return nil, ErrHijacked } @@ -710,31 +770,39 @@ func (c *conn) readRequest() (w *response, err error) { } return nil, err } + + if !http1ServerSupportsRequest(req) { + return nil, badRequestError("unsupported protocol version") + } + c.lastMethod = req.Method c.r.setInfiniteReadLimit() hosts, haveHost := req.Header["Host"] - if req.ProtoAtLeast(1, 1) && (!haveHost || len(hosts) == 0) { + isH2Upgrade := req.isH2Upgrade() + if req.ProtoAtLeast(1, 1) && (!haveHost || len(hosts) == 0) && !isH2Upgrade { return nil, badRequestError("missing required Host header") } if len(hosts) > 1 { return nil, badRequestError("too many Host headers") } - if len(hosts) == 1 && !validHostHeader(hosts[0]) { + if len(hosts) == 1 && !httplex.ValidHostHeader(hosts[0]) { return nil, badRequestError("malformed Host header") } for k, vv := range req.Header { - if !validHeaderName(k) { + if !httplex.ValidHeaderFieldName(k) { return nil, badRequestError("invalid header name") } for _, v := range vv { - if !validHeaderValue(v) { + if !httplex.ValidHeaderFieldValue(v) { return nil, badRequestError("invalid header value") } } } delete(req.Header, "Host") + ctx, cancelCtx := context.WithCancel(ctx) + req.ctx = ctx req.RemoteAddr = c.remoteAddr req.TLS = c.tlsState if body, ok := req.Body.(*body); ok { @@ -743,16 +811,43 @@ func (c *conn) readRequest() (w *response, err error) { w = &response{ conn: c, + cancelCtx: cancelCtx, req: req, reqBody: req.Body, handlerHeader: make(Header), contentLength: -1, + + // We populate these ahead of time so we're not + // reading from req.Header after their Handler starts + // and maybe mutates it (Issue 14940) + wants10KeepAlive: req.wantsHttp10KeepAlive(), + wantsClose: req.wantsClose(), + } + if isH2Upgrade { + w.closeAfterReply = true } w.cw.res = w w.w = newBufioWriterSize(&w.cw, bufferBeforeChunkingSize) return w, nil } +// http1ServerSupportsRequest reports whether Go's HTTP/1.x server +// supports the given request. +func http1ServerSupportsRequest(req *Request) bool { + if req.ProtoMajor == 1 { + return true + } + // Accept "PRI * HTTP/2.0" upgrade requests, so Handlers can + // wire up their own HTTP/2 upgrades. + if req.ProtoMajor == 2 && req.ProtoMinor == 0 && + req.Method == "PRI" && req.RequestURI == "*" { + return true + } + // Reject HTTP/0.x, and all other HTTP/2+ requests (which + // aren't encoded in ASCII anyway). + return false +} + func (w *response) Header() Header { if w.cw.header == nil && w.wroteHeader && !w.cw.wroteHeader { // Accessing the header between logically writing it @@ -766,7 +861,7 @@ func (w *response) Header() Header { // maxPostHandlerReadBytes is the max number of Request.Body bytes not // consumed by a handler that the server will read from the client -// in order to keep a connection alive. If there are more bytes than +// in order to keep a connection alive. If there are more bytes than // this then the server to be paranoid instead sends a "Connection: // close" response. // @@ -855,8 +950,8 @@ func (h extraHeader) Write(w *bufio.Writer) { // to cw.res.conn.bufw. // // p is not written by writeHeader, but is the first chunk of the body -// that will be written. It is sniffed for a Content-Type if none is -// set explicitly. It's also used to set the Content-Length, if the +// that will be written. It is sniffed for a Content-Type if none is +// set explicitly. It's also used to set the Content-Length, if the // total body size was small and the handler has already finished // running. func (cw *chunkWriter) writeHeader(p []byte) { @@ -911,9 +1006,9 @@ func (cw *chunkWriter) writeHeader(p []byte) { // Exceptions: 304/204/1xx responses never get Content-Length, and if // it was a HEAD request, we don't know the difference between // 0 actual bytes and 0 bytes because the handler noticed it - // was a HEAD request and chose not to write anything. So for + // was a HEAD request and chose not to write anything. So for // HEAD, the handler should either write the Content-Length or - // write non-zero bytes. If it's actually 0 bytes and the + // write non-zero bytes. If it's actually 0 bytes and the // handler never looked at the Request.Method, we just don't // send a Content-Length header. // Further, we don't send an automatic Content-Length if they @@ -925,7 +1020,7 @@ func (cw *chunkWriter) writeHeader(p []byte) { // If this was an HTTP/1.0 request with keep-alive and we sent a // Content-Length back, we can make this a keep-alive response ... - if w.req.wantsHttp10KeepAlive() && keepAlivesEnabled { + if w.wants10KeepAlive && keepAlivesEnabled { sentLength := header.get("Content-Length") != "" if sentLength && header.get("Connection") == "keep-alive" { w.closeAfterReply = false @@ -935,12 +1030,12 @@ func (cw *chunkWriter) writeHeader(p []byte) { // Check for a explicit (and valid) Content-Length header. hasCL := w.contentLength != -1 - if w.req.wantsHttp10KeepAlive() && (isHEAD || hasCL) { + if w.wants10KeepAlive && (isHEAD || hasCL || !bodyAllowedForStatus(w.status)) { _, connectionHeaderSet := header["Connection"] if !connectionHeaderSet { setHeader.connection = "keep-alive" } - } else if !w.req.ProtoAtLeast(1, 1) || w.req.wantsClose() { + } else if !w.req.ProtoAtLeast(1, 1) || w.wantsClose { w.closeAfterReply = true } @@ -965,9 +1060,12 @@ func (cw *chunkWriter) writeHeader(p []byte) { } // Per RFC 2616, we should consume the request body before - // replying, if the handler hasn't already done so. But we + // replying, if the handler hasn't already done so. But we // don't want to do an unbounded amount of reading here for // DoS reasons, so we only try up to a threshold. + // TODO(bradfitz): where does RFC 2616 say that? See Issue 15527 + // about HTTP/1.x Handlers concurrently reading and writing, like + // HTTP/2 handlers can do. Maybe this code should be relaxed? if w.req.ContentLength != 0 && !w.closeAfterReply { var discard, tooBig bool @@ -1009,7 +1107,7 @@ func (cw *chunkWriter) writeHeader(p []byte) { w.closeAfterReply = true } default: - // Some other kind of error occured, like a read timeout, or + // Some other kind of error occurred, like a read timeout, or // corrupt chunked encoding. In any case, whatever remains // on the wire must not be parsed as another HTTP request. w.closeAfterReply = true @@ -1069,6 +1167,10 @@ func (cw *chunkWriter) writeHeader(p []byte) { // to avoid closing the connection at EOF. cw.chunking = true setHeader.transferEncoding = "chunked" + if hasTE && te == "chunked" { + // We will send the chunked Transfer-Encoding header later. + delHeader("Transfer-Encoding") + } } } else { // HTTP version < 1.1: cannot do chunked transfer @@ -1148,7 +1250,7 @@ func statusLine(req *Request, code int) string { if proto11 { proto = "HTTP/1.1" } - codestring := strconv.Itoa(code) + codestring := fmt.Sprintf("%03d", code) text, ok := statusText[code] if !ok { text = "status code " + codestring @@ -1174,7 +1276,7 @@ func (w *response) bodyAllowed() bool { // The Life Of A Write is like this: // // Handler starts. No header has been sent. The handler can either -// write a header, or just start writing. Writing before sending a header +// write a header, or just start writing. Writing before sending a header // sends an implicitly empty 200 OK header. // // If the handler didn't declare a Content-Length up front, we either @@ -1200,7 +1302,7 @@ func (w *response) bodyAllowed() bool { // initial header contains both a Content-Type and Content-Length. // Also short-circuit in (1) when the header's been sent and not in // chunking mode, writing directly to (4) instead, if (2) has no -// buffered data. More generally, we could short-circuit from (1) to +// buffered data. More generally, we could short-circuit from (1) to // (3) even in chunking mode if the write size from (1) is over some // threshold and nothing is in (2). The answer might be mostly making // bufferBeforeChunkingSize smaller and having bufio's fast-paths deal @@ -1341,7 +1443,7 @@ type closeWriter interface { var _ closeWriter = (*net.TCPConn)(nil) // closeWrite flushes any outstanding data and sends a FIN packet (if -// client is connected via TCP), signalling that we're done. We then +// client is connected via TCP), signalling that we're done. We then // pause for a bit, hoping the client processes it before any // subsequent RST. // @@ -1355,7 +1457,7 @@ func (c *conn) closeWriteAndWait() { } // validNPN reports whether the proto is not a blacklisted Next -// Protocol Negotiation protocol. Empty and built-in protocol types +// Protocol Negotiation protocol. Empty and built-in protocol types // are blacklisted and can't be overridden with alternate // implementations. func validNPN(proto string) bool { @@ -1374,13 +1476,13 @@ func (c *conn) setState(nc net.Conn, state ConnState) { // badRequestError is a literal string (used by in the server in HTML, // unescaped) to tell the user why their request was bad. It should -// be plain text without user info or other embeddded errors. +// be plain text without user info or other embedded errors. type badRequestError string func (e badRequestError) Error() string { return "Bad Request: " + string(e) } // Serve a new connection. -func (c *conn) serve() { +func (c *conn) serve(ctx context.Context) { c.remoteAddr = c.rwc.RemoteAddr().String() defer func() { if err := recover(); err != nil { @@ -1417,12 +1519,17 @@ func (c *conn) serve() { } } + // HTTP/1.x from here on. + c.r = &connReader{r: c.rwc} c.bufr = newBufioReader(c.r) c.bufw = newBufioWriterSize(checkConnErrorWriter{c}, 4<<10) + ctx, cancelCtx := context.WithCancel(ctx) + defer cancelCtx() + for { - w, err := c.readRequest() + w, err := c.readRequest(ctx) if c.r.remain != c.server.initialReadLimitSize() { // If we read any bytes off the wire, we're active. c.setState(c.rwc, StateActive) @@ -1433,7 +1540,7 @@ func (c *conn) serve() { // able to read this if we're // responding to them and hanging up // while they're still writing their - // request. Undefined behavior. + // request. Undefined behavior. io.WriteString(c.rwc, "HTTP/1.1 431 Request Header Fields Too Large\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n431 Request Header Fields Too Large") c.closeWriteAndWait() return @@ -1467,9 +1574,10 @@ func (c *conn) serve() { // HTTP cannot have multiple simultaneous active requests.[*] // Until the server replies to this request, it can't read another, // so we might as well run the handler in this goroutine. - // [*] Not strictly true: HTTP pipelining. We could let them all process + // [*] Not strictly true: HTTP pipelining. We could let them all process // in parallel even if their responses need to be serialized. serverHandler{c.server}.ServeHTTP(w, w.req) + w.cancelCtx() if c.hijacked() { return } @@ -1488,7 +1596,7 @@ func (w *response) sendExpectationFailed() { // TODO(bradfitz): let ServeHTTP handlers handle // requests with non-standard expectation[s]? Seems // theoretical at best, and doesn't fit into the - // current ServeHTTP model anyway. We'd need to + // current ServeHTTP model anyway. We'd need to // make the ResponseWriter an optional // "ExpectReplier" interface or something. // @@ -1608,7 +1716,7 @@ func requestBodyRemains(rc io.ReadCloser) bool { } // The HandlerFunc type is an adapter to allow the use of -// ordinary functions as HTTP handlers. If f is a function +// ordinary functions as HTTP handlers. If f is a function // with the appropriate signature, HandlerFunc(f) is a // Handler that calls f. type HandlerFunc func(ResponseWriter, *Request) @@ -1621,6 +1729,8 @@ func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) { // Helper handlers // Error replies to the request with the specified error message and HTTP code. +// It does not otherwise end the request; the caller should ensure no further +// writes are done to w. // The error message should be plain text. func Error(w ResponseWriter, error string, code int) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") @@ -1709,7 +1819,7 @@ func Redirect(w ResponseWriter, r *Request, urlStr string, code int) { w.Header().Set("Location", urlStr) w.WriteHeader(code) - // RFC2616 recommends that a short note "SHOULD" be included in the + // RFC 2616 recommends that a short note "SHOULD" be included in the // response because older user agents may not understand 301/307. // Shouldn't send the response for POST or HEAD; that leaves GET. if r.Method == "GET" { @@ -1779,7 +1889,7 @@ func RedirectHandler(url string, code int) Handler { // been registered separately. // // Patterns may optionally begin with a host name, restricting matches to -// URLs on that host only. Host-specific patterns take precedence over +// URLs on that host only. Host-specific patterns take precedence over // general patterns, so that a handler might register for the two patterns // "/codesearch" and "codesearch.google.com/" without also taking over // requests for "http://www.google.com/". @@ -1800,10 +1910,12 @@ type muxEntry struct { } // NewServeMux allocates and returns a new ServeMux. -func NewServeMux() *ServeMux { return &ServeMux{m: make(map[string]muxEntry)} } +func NewServeMux() *ServeMux { return new(ServeMux) } // DefaultServeMux is the default ServeMux used by Serve. -var DefaultServeMux = NewServeMux() +var DefaultServeMux = &defaultServeMux + +var defaultServeMux ServeMux // Does path match pattern? func pathMatch(pattern, path string) bool { @@ -1926,6 +2038,9 @@ func (mux *ServeMux) Handle(pattern string, handler Handler) { panic("http: multiple registrations for " + pattern) } + if mux.m == nil { + mux.m = make(map[string]muxEntry) + } mux.m[pattern] = muxEntry{explicit: true, h: handler, pattern: pattern} if pattern[0] != '/' { @@ -1968,7 +2083,7 @@ func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { } // Serve accepts incoming HTTP connections on the listener l, -// creating a new service goroutine for each. The service goroutines +// creating a new service goroutine for each. The service goroutines // read requests and then call handler to reply to them. // Handler is typically nil, in which case the DefaultServeMux is used. func Serve(l net.Listener, handler Handler) error { @@ -1979,19 +2094,25 @@ func Serve(l net.Listener, handler Handler) error { // A Server defines parameters for running an HTTP server. // The zero value for Server is a valid configuration. type Server struct { - Addr string // TCP address to listen on, ":http" if empty - Handler Handler // handler to invoke, http.DefaultServeMux if nil - ReadTimeout time.Duration // maximum duration before timing out read of the request - WriteTimeout time.Duration // maximum duration before timing out write of the response - MaxHeaderBytes int // maximum size of request headers, DefaultMaxHeaderBytes if 0 - TLSConfig *tls.Config // optional TLS config, used by ListenAndServeTLS + Addr string // TCP address to listen on, ":http" if empty + Handler Handler // handler to invoke, http.DefaultServeMux if nil + ReadTimeout time.Duration // maximum duration before timing out read of the request + WriteTimeout time.Duration // maximum duration before timing out write of the response + TLSConfig *tls.Config // optional TLS config, used by ListenAndServeTLS + + // MaxHeaderBytes controls the maximum number of bytes the + // server will read parsing the request header's keys and + // values, including the request line. It does not limit the + // size of the request body. + // If zero, DefaultMaxHeaderBytes is used. + MaxHeaderBytes int // TLSNextProto optionally specifies a function to take over - // ownership of the provided TLS connection when an NPN - // protocol upgrade has occurred. The map key is the protocol + // ownership of the provided TLS connection when an NPN/ALPN + // protocol upgrade has occurred. The map key is the protocol // name negotiated. The Handler argument should be used to // handle HTTP requests and will initialize the Request's TLS - // and RemoteAddr if not already set. The connection is + // and RemoteAddr if not already set. The connection is // automatically closed when the function returns. // If TLSNextProto is nil, HTTP/2 support is enabled automatically. TLSNextProto map[string]func(*Server, *tls.Conn, Handler) @@ -2032,7 +2153,7 @@ const ( // For HTTP/2, StateActive fires on the transition from zero // to one active request, and only transitions away once all // active requests are complete. That means that ConnState - // can not be used to do per-request work; ConnState only notes + // cannot be used to do per-request work; ConnState only notes // the overall state of the connection. StateActive @@ -2100,9 +2221,37 @@ func (srv *Server) ListenAndServe() error { var testHookServerServe func(*Server, net.Listener) // used if non-nil +// shouldDoServeHTTP2 reports whether Server.Serve should configure +// automatic HTTP/2. (which sets up the srv.TLSNextProto map) +func (srv *Server) shouldConfigureHTTP2ForServe() bool { + if srv.TLSConfig == nil { + // Compatibility with Go 1.6: + // If there's no TLSConfig, it's possible that the user just + // didn't set it on the http.Server, but did pass it to + // tls.NewListener and passed that listener to Serve. + // So we should configure HTTP/2 (to set up srv.TLSNextProto) + // in case the listener returns an "h2" *tls.Conn. + return true + } + // The user specified a TLSConfig on their http.Server. + // In this, case, only configure HTTP/2 if their tls.Config + // explicitly mentions "h2". Otherwise http2.ConfigureServer + // would modify the tls.Config to add it, but they probably already + // passed this tls.Config to tls.NewListener. And if they did, + // it's too late anyway to fix it. It would only be potentially racy. + // See Issue 15908. + return strSliceContains(srv.TLSConfig.NextProtos, http2NextProtoTLS) +} + // Serve accepts incoming connections on the Listener l, creating a // new service goroutine for each. The service goroutines read requests and // then call srv.Handler to reply to them. +// +// For HTTP/2 support, srv.TLSConfig should be initialized to the +// provided listener's TLS Config before calling Serve. If +// srv.TLSConfig is non-nil and doesn't include the string "h2" in +// Config.NextProtos, HTTP/2 support is not enabled. +// // Serve always returns a non-nil error. func (srv *Server) Serve(l net.Listener) error { defer l.Close() @@ -2110,9 +2259,18 @@ func (srv *Server) Serve(l net.Listener) error { fn(srv, l) } var tempDelay time.Duration // how long to sleep on accept failure - if err := srv.setupHTTP2(); err != nil { - return err + + if srv.shouldConfigureHTTP2ForServe() { + if err := srv.setupHTTP2(); err != nil { + return err + } } + + // TODO: allow changing base context? can't imagine concrete + // use cases yet. + baseCtx := context.Background() + ctx := context.WithValue(baseCtx, ServerContextKey, srv) + ctx = context.WithValue(ctx, LocalAddrContextKey, l.Addr()) for { rw, e := l.Accept() if e != nil { @@ -2134,7 +2292,7 @@ func (srv *Server) Serve(l net.Listener) error { tempDelay = 0 c := srv.newConn(rw) c.setState(c.rwc, StateNew) // before Serve can return - go c.serve() + go c.serve(ctx) } } @@ -2309,15 +2467,10 @@ func (srv *Server) onceSetNextProtoDefaults() { // TimeoutHandler buffers all Handler writes to memory and does not // support the Hijacker or Flusher interfaces. func TimeoutHandler(h Handler, dt time.Duration, msg string) Handler { - t := time.NewTimer(dt) return &timeoutHandler{ handler: h, body: msg, - - // Effectively storing a *time.Timer, but decomposed - // for testing: - timeout: func() <-chan time.Time { return t.C }, - cancelTimer: t.Stop, + dt: dt, } } @@ -2328,12 +2481,11 @@ var ErrHandlerTimeout = errors.New("http: Handler timeout") type timeoutHandler struct { handler Handler body string + dt time.Duration - // timeout returns the channel of a *time.Timer and - // cancelTimer cancels it. They're stored separately for - // testing purposes. - timeout func() <-chan time.Time // returns channel producing a timeout - cancelTimer func() bool // optional + // When set, no timer will be created and this channel will + // be used instead. + testTimeout <-chan time.Time } func (h *timeoutHandler) errorBody() string { @@ -2344,6 +2496,12 @@ func (h *timeoutHandler) errorBody() string { } func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) { + var t *time.Timer + timeout := h.testTimeout + if timeout == nil { + t = time.NewTimer(h.dt) + timeout = t.C + } done := make(chan struct{}) tw := &timeoutWriter{ w: w, @@ -2361,12 +2519,15 @@ func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) { for k, vv := range tw.h { dst[k] = vv } + if !tw.wroteHeader { + tw.code = StatusOK + } w.WriteHeader(tw.code) w.Write(tw.wbuf.Bytes()) - if h.cancelTimer != nil { - h.cancelTimer() + if t != nil { + t.Stop() } - case <-h.timeout(): + case <-timeout: tw.mu.Lock() defer tw.mu.Unlock() w.WriteHeader(StatusServiceUnavailable) diff --git a/libgo/go/net/http/sniff.go b/libgo/go/net/http/sniff.go index 18810ba..0d21b44 100644 --- a/libgo/go/net/http/sniff.go +++ b/libgo/go/net/http/sniff.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -14,8 +14,8 @@ const sniffLen = 512 // DetectContentType implements the algorithm described // at http://mimesniff.spec.whatwg.org/ to determine the -// Content-Type of the given data. It considers at most the -// first 512 bytes of data. DetectContentType always returns +// Content-Type of the given data. It considers at most the +// first 512 bytes of data. DetectContentType always returns // a valid MIME type: if it cannot determine a more specific one, it // returns "application/octet-stream". func DetectContentType(data []byte) string { @@ -91,12 +91,41 @@ var sniffSignatures = []sniffSig{ ct: "image/webp", }, &exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"}, - &exactSig{[]byte("\x4F\x67\x67\x53\x00"), "application/ogg"}, &maskedSig{ mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), pat: []byte("RIFF\x00\x00\x00\x00WAVE"), ct: "audio/wave", }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), + pat: []byte("FORM\x00\x00\x00\x00AIFF"), + ct: "audio/aiff", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF"), + pat: []byte(".snd"), + ct: "audio/basic", + }, + &maskedSig{ + mask: []byte("OggS\x00"), + pat: []byte("\x4F\x67\x67\x53\x00"), + ct: "application/ogg", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"), + pat: []byte("MThd\x00\x00\x00\x06"), + ct: "audio/midi", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF"), + pat: []byte("ID3"), + ct: "audio/mpeg", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), + pat: []byte("RIFF\x00\x00\x00\x00AVI "), + ct: "video/avi", + }, &exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"}, &exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"}, &exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"}, @@ -126,9 +155,15 @@ type maskedSig struct { } func (m *maskedSig) match(data []byte, firstNonWS int) string { + // pattern matching algorithm section 6 + // https://mimesniff.spec.whatwg.org/#pattern-matching-algorithm + if m.skipWS { data = data[firstNonWS:] } + if len(m.pat) != len(m.mask) { + return "" + } if len(data) < len(m.mask) { return "" } diff --git a/libgo/go/net/http/sniff_test.go b/libgo/go/net/http/sniff_test.go index e008551..ac404bf 100644 --- a/libgo/go/net/http/sniff_test.go +++ b/libgo/go/net/http/sniff_test.go @@ -39,7 +39,18 @@ var sniffTests = []struct { {"GIF 87a", []byte(`GIF87a`), "image/gif"}, {"GIF 89a", []byte(`GIF89a...`), "image/gif"}, + // Audio types. + {"MIDI audio", []byte("MThd\x00\x00\x00\x06\x00\x01"), "audio/midi"}, + {"MP3 audio/MPEG audio", []byte("ID3\x03\x00\x00\x00\x00\x0f"), "audio/mpeg"}, + {"WAV audio #1", []byte("RIFFb\xb8\x00\x00WAVEfmt \x12\x00\x00\x00\x06"), "audio/wave"}, + {"WAV audio #2", []byte("RIFF,\x00\x00\x00WAVEfmt \x12\x00\x00\x00\x06"), "audio/wave"}, + {"AIFF audio #1", []byte("FORM\x00\x00\x00\x00AIFFCOMM\x00\x00\x00\x12\x00\x01\x00\x00\x57\x55\x00\x10\x40\x0d\xf3\x34"), "audio/aiff"}, + {"OGG audio", []byte("OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x7e\x46\x00\x00\x00\x00\x00\x00\x1f\xf6\xb4\xfc\x01\x1e\x01\x76\x6f\x72"), "application/ogg"}, + + // Video types. {"MP4 video", []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom<\x06t\xbfmdat"), "video/mp4"}, + {"AVI video #1", []byte("RIFF,O\n\x00AVI LISTÀ"), "video/avi"}, + {"AVI video #2", []byte("RIFF,\n\x00\x00AVI LISTÀ"), "video/avi"}, } func TestDetectContentType(t *testing.T) { diff --git a/libgo/go/net/http/status.go b/libgo/go/net/http/status.go index f3dacab..98645b7 100644 --- a/libgo/go/net/http/status.go +++ b/libgo/go/net/http/status.go @@ -4,63 +4,79 @@ package http -// HTTP status codes, defined in RFC 2616. +// HTTP status codes as registered with IANA. +// See: http://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml const ( - StatusContinue = 100 - StatusSwitchingProtocols = 101 + StatusContinue = 100 // RFC 7231, 6.2.1 + StatusSwitchingProtocols = 101 // RFC 7231, 6.2.2 + StatusProcessing = 102 // RFC 2518, 10.1 - StatusOK = 200 - StatusCreated = 201 - StatusAccepted = 202 - StatusNonAuthoritativeInfo = 203 - StatusNoContent = 204 - StatusResetContent = 205 - StatusPartialContent = 206 + StatusOK = 200 // RFC 7231, 6.3.1 + StatusCreated = 201 // RFC 7231, 6.3.2 + StatusAccepted = 202 // RFC 7231, 6.3.3 + StatusNonAuthoritativeInfo = 203 // RFC 7231, 6.3.4 + StatusNoContent = 204 // RFC 7231, 6.3.5 + StatusResetContent = 205 // RFC 7231, 6.3.6 + StatusPartialContent = 206 // RFC 7233, 4.1 + StatusMultiStatus = 207 // RFC 4918, 11.1 + StatusAlreadyReported = 208 // RFC 5842, 7.1 + StatusIMUsed = 226 // RFC 3229, 10.4.1 - StatusMultipleChoices = 300 - StatusMovedPermanently = 301 - StatusFound = 302 - StatusSeeOther = 303 - StatusNotModified = 304 - StatusUseProxy = 305 - StatusTemporaryRedirect = 307 + StatusMultipleChoices = 300 // RFC 7231, 6.4.1 + StatusMovedPermanently = 301 // RFC 7231, 6.4.2 + StatusFound = 302 // RFC 7231, 6.4.3 + StatusSeeOther = 303 // RFC 7231, 6.4.4 + StatusNotModified = 304 // RFC 7232, 4.1 + StatusUseProxy = 305 // RFC 7231, 6.4.5 + _ = 306 // RFC 7231, 6.4.6 (Unused) + StatusTemporaryRedirect = 307 // RFC 7231, 6.4.7 + StatusPermanentRedirect = 308 // RFC 7538, 3 - StatusBadRequest = 400 - StatusUnauthorized = 401 - StatusPaymentRequired = 402 - StatusForbidden = 403 - StatusNotFound = 404 - StatusMethodNotAllowed = 405 - StatusNotAcceptable = 406 - StatusProxyAuthRequired = 407 - StatusRequestTimeout = 408 - StatusConflict = 409 - StatusGone = 410 - StatusLengthRequired = 411 - StatusPreconditionFailed = 412 - StatusRequestEntityTooLarge = 413 - StatusRequestURITooLong = 414 - StatusUnsupportedMediaType = 415 - StatusRequestedRangeNotSatisfiable = 416 - StatusExpectationFailed = 417 - StatusTeapot = 418 - StatusPreconditionRequired = 428 - StatusTooManyRequests = 429 - StatusRequestHeaderFieldsTooLarge = 431 - StatusUnavailableForLegalReasons = 451 + StatusBadRequest = 400 // RFC 7231, 6.5.1 + StatusUnauthorized = 401 // RFC 7235, 3.1 + StatusPaymentRequired = 402 // RFC 7231, 6.5.2 + StatusForbidden = 403 // RFC 7231, 6.5.3 + StatusNotFound = 404 // RFC 7231, 6.5.4 + StatusMethodNotAllowed = 405 // RFC 7231, 6.5.5 + StatusNotAcceptable = 406 // RFC 7231, 6.5.6 + StatusProxyAuthRequired = 407 // RFC 7235, 3.2 + StatusRequestTimeout = 408 // RFC 7231, 6.5.7 + StatusConflict = 409 // RFC 7231, 6.5.8 + StatusGone = 410 // RFC 7231, 6.5.9 + StatusLengthRequired = 411 // RFC 7231, 6.5.10 + StatusPreconditionFailed = 412 // RFC 7232, 4.2 + StatusRequestEntityTooLarge = 413 // RFC 7231, 6.5.11 + StatusRequestURITooLong = 414 // RFC 7231, 6.5.12 + StatusUnsupportedMediaType = 415 // RFC 7231, 6.5.13 + StatusRequestedRangeNotSatisfiable = 416 // RFC 7233, 4.4 + StatusExpectationFailed = 417 // RFC 7231, 6.5.14 + StatusTeapot = 418 // RFC 7168, 2.3.3 + StatusUnprocessableEntity = 422 // RFC 4918, 11.2 + StatusLocked = 423 // RFC 4918, 11.3 + StatusFailedDependency = 424 // RFC 4918, 11.4 + StatusUpgradeRequired = 426 // RFC 7231, 6.5.15 + StatusPreconditionRequired = 428 // RFC 6585, 3 + StatusTooManyRequests = 429 // RFC 6585, 4 + StatusRequestHeaderFieldsTooLarge = 431 // RFC 6585, 5 + StatusUnavailableForLegalReasons = 451 // RFC 7725, 3 - StatusInternalServerError = 500 - StatusNotImplemented = 501 - StatusBadGateway = 502 - StatusServiceUnavailable = 503 - StatusGatewayTimeout = 504 - StatusHTTPVersionNotSupported = 505 - StatusNetworkAuthenticationRequired = 511 + StatusInternalServerError = 500 // RFC 7231, 6.6.1 + StatusNotImplemented = 501 // RFC 7231, 6.6.2 + StatusBadGateway = 502 // RFC 7231, 6.6.3 + StatusServiceUnavailable = 503 // RFC 7231, 6.6.4 + StatusGatewayTimeout = 504 // RFC 7231, 6.6.5 + StatusHTTPVersionNotSupported = 505 // RFC 7231, 6.6.6 + StatusVariantAlsoNegotiates = 506 // RFC 2295, 8.1 + StatusInsufficientStorage = 507 // RFC 4918, 11.5 + StatusLoopDetected = 508 // RFC 5842, 7.2 + StatusNotExtended = 510 // RFC 2774, 7 + StatusNetworkAuthenticationRequired = 511 // RFC 6585, 6 ) var statusText = map[int]string{ StatusContinue: "Continue", StatusSwitchingProtocols: "Switching Protocols", + StatusProcessing: "Processing", StatusOK: "OK", StatusCreated: "Created", @@ -69,6 +85,9 @@ var statusText = map[int]string{ StatusNoContent: "No Content", StatusResetContent: "Reset Content", StatusPartialContent: "Partial Content", + StatusMultiStatus: "Multi-Status", + StatusAlreadyReported: "Already Reported", + StatusIMUsed: "IM Used", StatusMultipleChoices: "Multiple Choices", StatusMovedPermanently: "Moved Permanently", @@ -77,6 +96,7 @@ var statusText = map[int]string{ StatusNotModified: "Not Modified", StatusUseProxy: "Use Proxy", StatusTemporaryRedirect: "Temporary Redirect", + StatusPermanentRedirect: "Permanent Redirect", StatusBadRequest: "Bad Request", StatusUnauthorized: "Unauthorized", @@ -97,6 +117,10 @@ var statusText = map[int]string{ StatusRequestedRangeNotSatisfiable: "Requested Range Not Satisfiable", StatusExpectationFailed: "Expectation Failed", StatusTeapot: "I'm a teapot", + StatusUnprocessableEntity: "Unprocessable Entity", + StatusLocked: "Locked", + StatusFailedDependency: "Failed Dependency", + StatusUpgradeRequired: "Upgrade Required", StatusPreconditionRequired: "Precondition Required", StatusTooManyRequests: "Too Many Requests", StatusRequestHeaderFieldsTooLarge: "Request Header Fields Too Large", @@ -108,6 +132,10 @@ var statusText = map[int]string{ StatusServiceUnavailable: "Service Unavailable", StatusGatewayTimeout: "Gateway Timeout", StatusHTTPVersionNotSupported: "HTTP Version Not Supported", + StatusVariantAlsoNegotiates: "Variant Also Negotiates", + StatusInsufficientStorage: "Insufficient Storage", + StatusLoopDetected: "Loop Detected", + StatusNotExtended: "Not Extended", StatusNetworkAuthenticationRequired: "Network Authentication Required", } diff --git a/libgo/go/net/http/transfer.go b/libgo/go/net/http/transfer.go index 6e59af8..c653467 100644 --- a/libgo/go/net/http/transfer.go +++ b/libgo/go/net/http/transfer.go @@ -17,6 +17,8 @@ import ( "strconv" "strings" "sync" + + "golang_org/x/net/lex/httplex" ) // ErrLineTooLong is returned when reading request or response bodies @@ -276,7 +278,7 @@ func (t *transferReader) protoAtLeast(m, n int) bool { } // bodyAllowedForStatus reports whether a given response status code -// permits a body. See RFC2616, section 4.4. +// permits a body. See RFC 2616, section 4.4. func bodyAllowedForStatus(status int) bool { switch { case status >= 100 && status <= 199: @@ -368,7 +370,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { // If there is no Content-Length or chunked Transfer-Encoding on a *Response // and the status is not 1xx, 204 or 304, then the body is unbounded. - // See RFC2616, section 4.4. + // See RFC 2616, section 4.4. switch msg.(type) { case *Response: if realLength == -1 && @@ -379,7 +381,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { } } - // Prepare body reader. ContentLength < 0 means chunked encoding + // Prepare body reader. ContentLength < 0 means chunked encoding // or close connection when finished, since multipart is not supported yet switch { case chunked(t.TransferEncoding): @@ -558,21 +560,19 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { if major < 1 { return true - } else if major == 1 && minor == 0 { - vv := header["Connection"] - if headerValuesContainsToken(vv, "close") || !headerValuesContainsToken(vv, "keep-alive") { - return true - } - return false - } else { - if headerValuesContainsToken(header["Connection"], "close") { - if removeCloseHeader { - header.Del("Connection") - } - return true - } } - return false + + conv := header["Connection"] + hasClose := httplex.HeaderValuesContainsToken(conv, "close") + if major == 1 && minor == 0 { + return hasClose || !httplex.HeaderValuesContainsToken(conv, "keep-alive") + } + + if hasClose && removeCloseHeader { + header.Del("Connection") + } + + return hasClose } // Parse the trailer header @@ -729,11 +729,11 @@ func (b *body) readTrailer() error { } // Make sure there's a header terminator coming up, to prevent - // a DoS with an unbounded size Trailer. It's not easy to + // a DoS with an unbounded size Trailer. It's not easy to // slip in a LimitReader here, as textproto.NewReader requires - // a concrete *bufio.Reader. Also, we can't get all the way + // a concrete *bufio.Reader. Also, we can't get all the way // back up to our conn's LimitedReader that *might* be backing - // this bufio.Reader. Instead, a hack: we iteratively Peek up + // this bufio.Reader. Instead, a hack: we iteratively Peek up // to the bufio.Reader's max size, looking for a double CRLF. // This limits the trailer to the underlying buffer size, typically 4kB. if !seeUpcomingDoubleCRLF(b.r) { diff --git a/libgo/go/net/http/transport.go b/libgo/go/net/http/transport.go index baf71d5..9164d0d 100644 --- a/libgo/go/net/http/transport.go +++ b/libgo/go/net/http/transport.go @@ -12,17 +12,22 @@ package http import ( "bufio" "compress/gzip" + "container/list" + "context" "crypto/tls" "errors" "fmt" "io" "log" "net" + "net/http/httptrace" "net/url" "os" "strings" "sync" "time" + + "golang_org/x/net/lex/httplex" ) // DefaultTransport is the default implementation of Transport and is @@ -32,10 +37,12 @@ import ( // $no_proxy) environment variables. var DefaultTransport RoundTripper = &Transport{ Proxy: ProxyFromEnvironment, - Dial: (&net.Dialer{ + DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, - }).Dial, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } @@ -63,9 +70,10 @@ const DefaultMaxIdleConnsPerHost = 2 // See the package docs for more about HTTP/2. type Transport struct { idleMu sync.Mutex - wantIdle bool // user has requested to close all idle conns - idleConn map[connectMethodKey][]*persistConn + wantIdle bool // user has requested to close all idle conns + idleConn map[connectMethodKey][]*persistConn // most recently used at end idleConnCh map[connectMethodKey]chan *persistConn + idleLRU connLRU reqMu sync.Mutex reqCanceler map[*Request]func() @@ -79,9 +87,16 @@ type Transport struct { // If Proxy is nil or returns a nil *URL, no proxy is used. Proxy func(*Request) (*url.URL, error) - // Dial specifies the dial function for creating unencrypted - // TCP connections. - // If Dial is nil, net.Dial is used. + // DialContext specifies the dial function for creating unencrypted TCP connections. + // If DialContext is nil (and the deprecated Dial below is also nil), + // then the transport dials using package net. + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) + + // Dial specifies the dial function for creating unencrypted TCP connections. + // + // Deprecated: Use DialContext instead, which allows the transport + // to cancel dials as soon as they are no longer needed. + // If both are set, DialContext takes priority. Dial func(network, addr string) (net.Conn, error) // DialTLS specifies an optional dial function for creating @@ -117,11 +132,21 @@ type Transport struct { // uncompressed. DisableCompression bool + // MaxIdleConns controls the maximum number of idle (keep-alive) + // connections across all hosts. Zero means no limit. + MaxIdleConns int + // MaxIdleConnsPerHost, if non-zero, controls the maximum idle - // (keep-alive) to keep per-host. If zero, + // (keep-alive) connections to keep per-host. If zero, // DefaultMaxIdleConnsPerHost is used. MaxIdleConnsPerHost int + // IdleConnTimeout is the maximum amount of time an idle + // (keep-alive) connection will remain idle before closing + // itself. + // Zero means no limit. + IdleConnTimeout time.Duration + // ResponseHeaderTimeout, if non-zero, specifies the amount of // time to wait for a server's response headers after fully // writing the request (including its body, if any). This @@ -137,7 +162,7 @@ type Transport struct { // TLSNextProto specifies how the Transport switches to an // alternate protocol (such as HTTP/2) after a TLS NPN/ALPN - // protocol negotiation. If Transport dials an TLS connection + // protocol negotiation. If Transport dials an TLS connection // with a non-empty protocol name and TLSNextProto contains a // map entry for that key (such as "h2"), then the func is // called with the request's authority (such as "example.com" @@ -146,13 +171,18 @@ type Transport struct { // If TLSNextProto is nil, HTTP/2 support is enabled automatically. TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper + // MaxResponseHeaderBytes specifies a limit on how many + // response bytes are allowed in the server's response + // header. + // + // Zero means to use a default limit. + MaxResponseHeaderBytes int64 + // nextProtoOnce guards initialization of TLSNextProto and // h2transport (via onceSetNextProtoDefaults) nextProtoOnce sync.Once h2transport *http2Transport // non-nil if http2 wired up - // TODO: tunable on global max cached connections - // TODO: tunable on timeout on cached connections // TODO: tunable on max per-host TCP dials in flight (Issue 13957) } @@ -167,25 +197,34 @@ func (t *Transport) onceSetNextProtoDefaults() { // Transport. return } - if t.TLSClientConfig != nil { - // Be conservative for now (for Go 1.6) at least and - // don't automatically enable http2 if they've - // specified a custom TLS config. Let them opt-in - // themselves via http2.ConfigureTransport so we don't - // surprise them by modifying their tls.Config. - // Issue 14275. - return - } - if t.ExpectContinueTimeout != 0 { - // Unsupported in http2, so disable http2 for now. - // Issue 13851. + if t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil { + // Be conservative and don't automatically enable + // http2 if they've specified a custom TLS config or + // custom dialers. Let them opt-in themselves via + // http2.ConfigureTransport so we don't surprise them + // by modifying their tls.Config. Issue 14275. return } t2, err := http2configureTransport(t) if err != nil { log.Printf("Error enabling Transport HTTP/2 support: %v", err) - } else { - t.h2transport = t2 + return + } + t.h2transport = t2 + + // Auto-configure the http2.Transport's MaxHeaderListSize from + // the http.Transport's MaxResponseHeaderBytes. They don't + // exactly mean the same thing, but they're close. + // + // TODO: also add this to x/net/http2.Configure Transport, behind + // a +build go1.7 build tag: + if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 { + const h2max = 1<<32 - 1 + if limit1 >= h2max { + t2.MaxHeaderListSize = h2max + } else { + t2.MaxHeaderListSize = uint32(limit1) + } } } @@ -212,6 +251,9 @@ func ProxyFromEnvironment(req *Request) (*url.URL, error) { } if proxy == "" { proxy = httpProxyEnv.Get() + if proxy != "" && os.Getenv("REQUEST_METHOD") != "" { + return nil, errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") + } } if proxy == "" { return nil, nil @@ -245,8 +287,9 @@ func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { // transportRequest is a wrapper around a *Request that adds // optional extra headers to write. type transportRequest struct { - *Request // original request, not to be mutated - extra Header // extra headers to write, or nil + *Request // original request, not to be mutated + extra Header // extra headers to write, or nil + trace *httptrace.ClientTrace // optional } func (tr *transportRequest) extraHeaders() Header { @@ -262,6 +305,9 @@ func (tr *transportRequest) extraHeaders() Header { // and redirects), see Get, Post, and the Client type. func (t *Transport) RoundTrip(req *Request) (*Response, error) { t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + ctx := req.Context() + trace := httptrace.ContextClientTrace(ctx) + if req.URL == nil { req.closeBody() return nil, errors.New("http: nil Request.URL") @@ -270,18 +316,32 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { req.closeBody() return nil, errors.New("http: nil Request.Header") } + scheme := req.URL.Scheme + isHTTP := scheme == "http" || scheme == "https" + if isHTTP { + for k, vv := range req.Header { + if !httplex.ValidHeaderFieldName(k) { + return nil, fmt.Errorf("net/http: invalid header field name %q", k) + } + for _, v := range vv { + if !httplex.ValidHeaderFieldValue(v) { + return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k) + } + } + } + } // TODO(bradfitz): switch to atomic.Value for this map instead of RWMutex t.altMu.RLock() - altRT := t.altProto[req.URL.Scheme] + altRT := t.altProto[scheme] t.altMu.RUnlock() if altRT != nil { if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { return resp, err } } - if s := req.URL.Scheme; s != "http" && s != "https" { + if !isHTTP { req.closeBody() - return nil, &badStringError{"unsupported protocol scheme", s} + return nil, &badStringError{"unsupported protocol scheme", scheme} } if req.Method != "" && !validMethod(req.Method) { return nil, fmt.Errorf("net/http: invalid method %q", req.Method) @@ -293,7 +353,7 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { for { // treq gets modified by roundTrip, so we need to recreate for each retry. - treq := &transportRequest{Request: req} + treq := &transportRequest{Request: req, trace: trace} cm, err := t.connectMethodForRequest(treq) if err != nil { req.closeBody() @@ -302,9 +362,9 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { // Get the cached or newly-created connection to either the // host (for http or https), the http proxy, or the http proxy - // pre-CONNECTed to https server. In any case, we'll be ready + // pre-CONNECTed to https server. In any case, we'll be ready // to send it requests. - pconn, err := t.getConn(req, cm) + pconn, err := t.getConn(treq, cm) if err != nil { t.setReqCanceler(req, nil) req.closeBody() @@ -322,46 +382,47 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { if err == nil { return resp, nil } - if err := checkTransportResend(err, req, pconn); err != nil { + if !pconn.shouldRetryRequest(req, err) { return nil, err } testHookRoundTripRetried() } } -// checkTransportResend checks whether a failed HTTP request can be -// resent on a new connection. The non-nil input error is the error from -// roundTrip, which might be wrapped in a beforeRespHeaderError error. -// -// The return value is err or the unwrapped error inside a -// beforeRespHeaderError. -func checkTransportResend(err error, req *Request, pconn *persistConn) error { - brhErr, ok := err.(beforeRespHeaderError) - if !ok { - return err +// shouldRetryRequest reports whether we should retry sending a failed +// HTTP request on a new connection. The non-nil input error is the +// error from roundTrip. +func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool { + if err == errMissingHost { + // User error. + return false } - err = brhErr.error // unwrap the custom error in case we return it - if err != errMissingHost && pconn.isReused() && req.isReplayable() { - // If we try to reuse a connection that the server is in the process of - // closing, we may end up successfully writing out our request (or a - // portion of our request) only to find a connection error when we try to - // read from (or finish writing to) the socket. - - // There can be a race between the socket pool checking whether a socket - // is still connected, receiving the FIN, and sending/reading data on a - // reused socket. If we receive the FIN between the connectedness check - // and writing/reading from the socket, we may first learn the socket is - // disconnected when we get a ERR_SOCKET_NOT_CONNECTED. This will most - // likely happen when trying to retrieve its IP address. See - // http://crbug.com/105824 for more details. - - // We resend a request only if we reused a keep-alive connection and did - // not yet receive any header data. This automatically prevents an - // infinite resend loop because we'll run out of the cached keep-alive - // connections eventually. - return nil + if !pc.isReused() { + // This was a fresh connection. There's no reason the server + // should've hung up on us. + // + // Also, if we retried now, we could loop forever + // creating new connections and retrying if the server + // is just hanging up on us because it doesn't like + // our request (as opposed to sending an error). + return false } - return err + if !req.isReplayable() { + // Don't retry non-idempotent requests. + + // TODO: swap the nothingWrittenError and isReplayable checks, + // putting the "if nothingWrittenError => return true" case + // first, per golang.org/issue/15723 + return false + } + if _, ok := err.(nothingWrittenError); ok { + // We never wrote anything, so it's safe to retry. + return true + } + if err == errServerClosedIdle || err == errServerClosedConn { + return true + } + return false // conservatively } // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. @@ -400,6 +461,7 @@ func (t *Transport) CloseIdleConnections() { t.idleConn = nil t.idleConnCh = nil t.wantIdle = true + t.idleLRU = connLRU{} t.idleMu.Unlock() for _, conns := range m { for _, pconn := range conns { @@ -414,7 +476,7 @@ func (t *Transport) CloseIdleConnections() { // CancelRequest cancels an in-flight request by closing its connection. // CancelRequest should only be called after RoundTrip has returned. // -// Deprecated: Use Request.Cancel instead. CancelRequest can not cancel +// Deprecated: Use Request.Cancel instead. CancelRequest cannot cancel // HTTP/2 requests. func (t *Transport) CancelRequest(req *Request) { t.reqMu.Lock() @@ -500,9 +562,12 @@ var ( errConnBroken = errors.New("http: putIdleConn: connection is in bad state") errWantIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") + errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host") errCloseIdleConns = errors.New("http: CloseIdleConnections called") errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") - errServerClosedIdle = errors.New("http: server closed idle conn") + errServerClosedIdle = errors.New("http: server closed idle connection") + errServerClosedConn = errors.New("http: server closed connection") + errIdleConnTimeout = errors.New("http: idle connection timeout") ) func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { @@ -511,6 +576,13 @@ func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { } } +func (t *Transport) maxIdleConnsPerHost() int { + if v := t.MaxIdleConnsPerHost; v != 0 { + return v + } + return DefaultMaxIdleConnsPerHost +} + // tryPutIdleConn adds pconn to the list of idle persistent connections awaiting // a new request. // If pconn is no longer needed or not in a good state, tryPutIdleConn returns @@ -523,13 +595,11 @@ func (t *Transport) tryPutIdleConn(pconn *persistConn) error { if pconn.isBroken() { return errConnBroken } - key := pconn.cacheKey - max := t.MaxIdleConnsPerHost - if max == 0 { - max = DefaultMaxIdleConnsPerHost - } pconn.markReused() + key := pconn.cacheKey + t.idleMu.Lock() + defer t.idleMu.Unlock() waitingDialer := t.idleConnCh[key] select { @@ -537,9 +607,8 @@ func (t *Transport) tryPutIdleConn(pconn *persistConn) error { // We're done with this pconn and somebody else is // currently waiting for a conn of this type (they're // actively dialing, but this conn is ready - // first). Chrome calls this socket late binding. See + // first). Chrome calls this socket late binding. See // https://insouciant.org/tech/connection-management-in-chromium/ - t.idleMu.Unlock() return nil default: if waitingDialer != nil { @@ -549,23 +618,35 @@ func (t *Transport) tryPutIdleConn(pconn *persistConn) error { } } if t.wantIdle { - t.idleMu.Unlock() return errWantIdle } if t.idleConn == nil { t.idleConn = make(map[connectMethodKey][]*persistConn) } - if len(t.idleConn[key]) >= max { - t.idleMu.Unlock() - return errTooManyIdle + idles := t.idleConn[key] + if len(idles) >= t.maxIdleConnsPerHost() { + return errTooManyIdleHost } - for _, exist := range t.idleConn[key] { + for _, exist := range idles { if exist == pconn { log.Fatalf("dup idle pconn %p in freelist", pconn) } } - t.idleConn[key] = append(t.idleConn[key], pconn) - t.idleMu.Unlock() + t.idleConn[key] = append(idles, pconn) + t.idleLRU.add(pconn) + if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns { + oldest := t.idleLRU.removeOldest() + oldest.close(errTooManyIdle) + t.removeIdleConnLocked(oldest) + } + if t.IdleConnTimeout > 0 { + if pconn.idleTimer != nil { + pconn.idleTimer.Reset(t.IdleConnTimeout) + } else { + pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) + } + } + pconn.idleAt = time.Now() return nil } @@ -591,29 +672,75 @@ func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn { return ch } -func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) { +func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn, idleSince time.Time) { key := cm.key() t.idleMu.Lock() defer t.idleMu.Unlock() - if t.idleConn == nil { - return nil - } for { pconns, ok := t.idleConn[key] if !ok { - return nil + return nil, time.Time{} } if len(pconns) == 1 { pconn = pconns[0] delete(t.idleConn, key) } else { - // 2 or more cached connections; pop last - // TODO: queue? + // 2 or more cached connections; use the most + // recently used one at the end. pconn = pconns[len(pconns)-1] t.idleConn[key] = pconns[:len(pconns)-1] } - if !pconn.isBroken() { - return + t.idleLRU.remove(pconn) + if pconn.isBroken() { + // There is a tiny window where this is + // possible, between the connecting dying and + // the persistConn readLoop calling + // Transport.removeIdleConn. Just skip it and + // carry on. + continue + } + if pconn.idleTimer != nil && !pconn.idleTimer.Stop() { + // We picked this conn at the ~same time it + // was expiring and it's trying to close + // itself in another goroutine. Don't use it. + continue + } + return pconn, pconn.idleAt + } +} + +// removeIdleConn marks pconn as dead. +func (t *Transport) removeIdleConn(pconn *persistConn) { + t.idleMu.Lock() + defer t.idleMu.Unlock() + t.removeIdleConnLocked(pconn) +} + +// t.idleMu must be held. +func (t *Transport) removeIdleConnLocked(pconn *persistConn) { + if pconn.idleTimer != nil { + pconn.idleTimer.Stop() + } + t.idleLRU.remove(pconn) + key := pconn.cacheKey + pconns, _ := t.idleConn[key] + switch len(pconns) { + case 0: + // Nothing + case 1: + if pconns[0] == pconn { + delete(t.idleConn, key) + } + default: + for i, v := range pconns { + if v != pconn { + continue + } + // Slide down, keeping most recently-used + // conns at the end. + copy(pconns[i:], pconns[i+1:]) + t.idleConn[key] = pconns[:len(pconns)-1] + break } } } @@ -650,7 +777,12 @@ func (t *Transport) replaceReqCanceler(r *Request, fn func()) bool { return true } -func (t *Transport) dial(network, addr string) (net.Conn, error) { +var zeroDialer net.Dialer + +func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { + if t.DialContext != nil { + return t.DialContext(ctx, network, addr) + } if t.Dial != nil { c, err := t.Dial(network, addr) if c == nil && err == nil { @@ -658,15 +790,24 @@ func (t *Transport) dial(network, addr string) (net.Conn, error) { } return c, err } - return net.Dial(network, addr) + return zeroDialer.DialContext(ctx, network, addr) } // getConn dials and creates a new persistConn to the target as -// specified in the connectMethod. This includes doing a proxy CONNECT +// specified in the connectMethod. This includes doing a proxy CONNECT // and/or setting up TLS. If this doesn't return an error, the persistConn // is ready to write requests to. -func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) { - if pc := t.getIdleConn(cm); pc != nil { +func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistConn, error) { + req := treq.Request + trace := treq.trace + ctx := req.Context() + if trace != nil && trace.GetConn != nil { + trace.GetConn(cm.addr()) + } + if pc, idleSince := t.getIdleConn(cm); pc != nil { + if trace != nil && trace.GotConn != nil { + trace.GotConn(pc.gotIdleConnTrace(idleSince)) + } // set request canceler to some non-nil function so we // can detect whether it was cleared between now and when // we enter roundTrip @@ -699,7 +840,7 @@ func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error t.setReqCanceler(req, func() { close(cancelc) }) go func() { - pc, err := t.dialConn(cm) + pc, err := t.dialConn(ctx, cm) dialc <- dialRes{pc, err} }() @@ -707,7 +848,26 @@ func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error select { case v := <-dialc: // Our dial finished. - return v.pc, v.err + if v.pc != nil { + if trace != nil && trace.GotConn != nil && v.pc.alt == nil { + trace.GotConn(httptrace.GotConnInfo{Conn: v.pc.conn}) + } + return v.pc, nil + } + // Our dial failed. See why to return a nicer error + // value. + select { + case <-req.Cancel: + case <-req.Context().Done(): + case <-cancelc: + default: + // It wasn't an error due to cancelation, so + // return the original error message: + return nil, v.err + } + // It was an error due to cancelation, so prioritize that + // error value. (Issue 16049) + return nil, errRequestCanceledConn case pc := <-idleConnCh: // Another request finished first and its net.Conn // became available before our dial. Or somebody @@ -715,24 +875,31 @@ func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error // But our dial is still going, so give it away // when it finishes: handlePendingDial() + if trace != nil && trace.GotConn != nil { + trace.GotConn(httptrace.GotConnInfo{Conn: pc.conn, Reused: pc.isReused()}) + } return pc, nil case <-req.Cancel: handlePendingDial() return nil, errRequestCanceledConn + case <-req.Context().Done(): + handlePendingDial() + return nil, errRequestCanceledConn case <-cancelc: handlePendingDial() return nil, errRequestCanceledConn } } -func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { +func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistConn, error) { pconn := &persistConn{ - t: t, - cacheKey: cm.key(), - reqch: make(chan requestAndChan, 1), - writech: make(chan writeRequest, 1), - closech: make(chan struct{}), - writeErrCh: make(chan error, 1), + t: t, + cacheKey: cm.key(), + reqch: make(chan requestAndChan, 1), + writech: make(chan writeRequest, 1), + closech: make(chan struct{}), + writeErrCh: make(chan error, 1), + writeLoopDone: make(chan struct{}), } tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil if tlsDial { @@ -755,7 +922,7 @@ func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { pconn.tlsState = &cs } } else { - conn, err := t.dial("tcp", cm.addr()) + conn, err := t.dial(ctx, "tcp", cm.addr()) if err != nil { if cm.proxyURL != nil { err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err) @@ -848,13 +1015,29 @@ func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { } } - pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF}) - pconn.bw = bufio.NewWriter(pconn.conn) + pconn.br = bufio.NewReader(pconn) + pconn.bw = bufio.NewWriter(persistConnWriter{pconn}) go pconn.readLoop() go pconn.writeLoop() return pconn, nil } +// persistConnWriter is the io.Writer written to by pc.bw. +// It accumulates the number of bytes written to the underlying conn, +// so the retry logic can determine whether any bytes made it across +// the wire. +// This is exactly 1 pointer field wide so it can go into an interface +// without allocation. +type persistConnWriter struct { + pc *persistConn +} + +func (w persistConnWriter) Write(p []byte) (n int, err error) { + n, err = w.pc.conn.Write(p) + w.pc.nwrite += int64(n) + return +} + // useProxy reports whether requests to addr should use a proxy, // according to the NO_PROXY or no_proxy environment variable. // addr is always a canonicalAddr with a host and port. @@ -978,28 +1161,36 @@ func (k connectMethodKey) String() string { // (but may be used for non-keep-alive requests as well) type persistConn struct { // alt optionally specifies the TLS NextProto RoundTripper. - // This is used for HTTP/2 today and future protocol laters. + // This is used for HTTP/2 today and future protocols later. // If it's non-nil, the rest of the fields are unused. alt RoundTripper - t *Transport - cacheKey connectMethodKey - conn net.Conn - tlsState *tls.ConnectionState - br *bufio.Reader // from conn - sawEOF bool // whether we've seen EOF from conn; owned by readLoop - bw *bufio.Writer // to conn - reqch chan requestAndChan // written by roundTrip; read by readLoop - writech chan writeRequest // written by roundTrip; read by writeLoop - closech chan struct{} // closed when conn closed - isProxy bool + t *Transport + cacheKey connectMethodKey + conn net.Conn + tlsState *tls.ConnectionState + br *bufio.Reader // from conn + bw *bufio.Writer // to conn + nwrite int64 // bytes written + reqch chan requestAndChan // written by roundTrip; read by readLoop + writech chan writeRequest // written by roundTrip; read by writeLoop + closech chan struct{} // closed when conn closed + isProxy bool + sawEOF bool // whether we've seen EOF from conn; owned by readLoop + readLimit int64 // bytes allowed to be read; owned by readLoop // writeErrCh passes the request write error (usually nil) // from the writeLoop goroutine to the readLoop which passes // it off to the res.Body reader, which then uses it to decide // whether or not a connection can be reused. Issue 7569. writeErrCh chan error - lk sync.Mutex // guards following fields + writeLoopDone chan struct{} // closed when write loop ends + + // Both guarded by Transport.idleMu: + idleAt time.Time // time it last become idle + idleTimer *time.Timer // holding an AfterFunc to close it + + mu sync.Mutex // guards following fields numExpectedResponses int closed error // set non-nil when conn is closed, before closech is closed broken bool // an error has happened on this connection; marked broken so it's not reused. @@ -1011,45 +1202,153 @@ type persistConn struct { mutateHeaderFunc func(Header) } +func (pc *persistConn) maxHeaderResponseSize() int64 { + if v := pc.t.MaxResponseHeaderBytes; v != 0 { + return v + } + return 10 << 20 // conservative default; same as http2 +} + +func (pc *persistConn) Read(p []byte) (n int, err error) { + if pc.readLimit <= 0 { + return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize()) + } + if int64(len(p)) > pc.readLimit { + p = p[:pc.readLimit] + } + n, err = pc.conn.Read(p) + if err == io.EOF { + pc.sawEOF = true + } + pc.readLimit -= int64(n) + return +} + // isBroken reports whether this connection is in a known broken state. func (pc *persistConn) isBroken() bool { - pc.lk.Lock() - b := pc.broken - pc.lk.Unlock() + pc.mu.Lock() + b := pc.closed != nil + pc.mu.Unlock() return b } // isCanceled reports whether this connection was closed due to CancelRequest. func (pc *persistConn) isCanceled() bool { - pc.lk.Lock() - defer pc.lk.Unlock() + pc.mu.Lock() + defer pc.mu.Unlock() return pc.canceled } // isReused reports whether this connection is in a known broken state. func (pc *persistConn) isReused() bool { - pc.lk.Lock() + pc.mu.Lock() r := pc.reused - pc.lk.Unlock() + pc.mu.Unlock() return r } +func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) { + pc.mu.Lock() + defer pc.mu.Unlock() + t.Reused = pc.reused + t.Conn = pc.conn + t.WasIdle = true + if !idleAt.IsZero() { + t.IdleTime = time.Since(idleAt) + } + return +} + func (pc *persistConn) cancelRequest() { - pc.lk.Lock() - defer pc.lk.Unlock() + pc.mu.Lock() + defer pc.mu.Unlock() pc.canceled = true pc.closeLocked(errRequestCanceled) } +// closeConnIfStillIdle closes the connection if it's still sitting idle. +// This is what's called by the persistConn's idleTimer, and is run in its +// own goroutine. +func (pc *persistConn) closeConnIfStillIdle() { + t := pc.t + t.idleMu.Lock() + defer t.idleMu.Unlock() + if _, ok := t.idleLRU.m[pc]; !ok { + // Not idle. + return + } + t.removeIdleConnLocked(pc) + pc.close(errIdleConnTimeout) +} + +// mapRoundTripErrorFromReadLoop maps the provided readLoop error into +// the error value that should be returned from persistConn.roundTrip. +// +// The startBytesWritten value should be the value of pc.nwrite before the roundTrip +// started writing the request. +func (pc *persistConn) mapRoundTripErrorFromReadLoop(startBytesWritten int64, err error) (out error) { + if err == nil { + return nil + } + if pc.isCanceled() { + return errRequestCanceled + } + if err == errServerClosedIdle || err == errServerClosedConn { + return err + } + if pc.isBroken() { + <-pc.writeLoopDone + if pc.nwrite == startBytesWritten { + return nothingWrittenError{err} + } + } + return err +} + +// mapRoundTripErrorAfterClosed returns the error value to be propagated +// up to Transport.RoundTrip method when persistConn.roundTrip sees +// its pc.closech channel close, indicating the persistConn is dead. +// (after closech is closed, pc.closed is valid). +func (pc *persistConn) mapRoundTripErrorAfterClosed(startBytesWritten int64) error { + if pc.isCanceled() { + return errRequestCanceled + } + err := pc.closed + if err == errServerClosedIdle || err == errServerClosedConn { + // Don't decorate + return err + } + + // Wait for the writeLoop goroutine to terminated, and then + // see if we actually managed to write anything. If not, we + // can retry the request. + <-pc.writeLoopDone + if pc.nwrite == startBytesWritten { + return nothingWrittenError{err} + } + + return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err) + +} + func (pc *persistConn) readLoop() { closeErr := errReadLoopExiting // default value, if not changed below - defer func() { pc.close(closeErr) }() + defer func() { + pc.close(closeErr) + pc.t.removeIdleConn(pc) + }() - tryPutIdleConn := func() bool { + tryPutIdleConn := func(trace *httptrace.ClientTrace) bool { if err := pc.t.tryPutIdleConn(pc); err != nil { closeErr = err + if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled { + trace.PutIdleConn(err) + } return false } + if trace != nil && trace.PutIdleConn != nil { + trace.PutIdleConn(nil) + } return true } @@ -1066,27 +1365,33 @@ func (pc *persistConn) readLoop() { alive := true for alive { + pc.readLimit = pc.maxHeaderResponseSize() _, err := pc.br.Peek(1) - if err != nil { - err = beforeRespHeaderError{err} - } - pc.lk.Lock() + pc.mu.Lock() if pc.numExpectedResponses == 0 { pc.readLoopPeekFailLocked(err) - pc.lk.Unlock() + pc.mu.Unlock() return } - pc.lk.Unlock() + pc.mu.Unlock() rc := <-pc.reqch + trace := httptrace.ContextClientTrace(rc.req.Context()) var resp *Response if err == nil { - resp, err = pc.readResponse(rc) + resp, err = pc.readResponse(rc, trace) + } else { + err = errServerClosedConn + closeErr = err } if err != nil { + if pc.readLimit <= 0 { + err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) + } + // If we won't be able to retry this request later (from the // roundTrip goroutine), mark it as done now. // BEFORE the send on rc.ch, as the client might re-use the @@ -1094,7 +1399,7 @@ func (pc *persistConn) readLoop() { // t.setReqCanceler from this persistConn while the Transport // potentially spins up a different persistConn for the // caller's subsequent request. - if checkTransportResend(err, rc.req, pc) != nil { + if !pc.shouldRetryRequest(rc.req, err) { pc.t.setReqCanceler(rc.req, nil) } select { @@ -1104,10 +1409,11 @@ func (pc *persistConn) readLoop() { } return } + pc.readLimit = maxInt64 // effictively no limit for response bodies - pc.lk.Lock() + pc.mu.Lock() pc.numExpectedResponses-- - pc.lk.Unlock() + pc.mu.Unlock() hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 @@ -1130,7 +1436,7 @@ func (pc *persistConn) readLoop() { alive = alive && !pc.sawEOF && pc.wroteRequest() && - tryPutIdleConn() + tryPutIdleConn(trace) select { case rc.ch <- responseAndError{res: resp}: @@ -1145,25 +1451,33 @@ func (pc *persistConn) readLoop() { continue } - if rc.addedGzip { - maybeUngzipResponse(resp) + waitForBodyRead := make(chan bool, 2) + body := &bodyEOFSignal{ + body: resp.Body, + earlyCloseFn: func() error { + waitForBodyRead <- false + return nil + + }, + fn: func(err error) error { + isEOF := err == io.EOF + waitForBodyRead <- isEOF + if isEOF { + <-eofc // see comment above eofc declaration + } else if err != nil && pc.isCanceled() { + return errRequestCanceled + } + return err + }, } - resp.Body = &bodyEOFSignal{body: resp.Body} - waitForBodyRead := make(chan bool, 2) - resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error { - waitForBodyRead <- false - return nil - } - resp.Body.(*bodyEOFSignal).fn = func(err error) error { - isEOF := err == io.EOF - waitForBodyRead <- isEOF - if isEOF { - <-eofc // see comment above eofc declaration - } else if err != nil && pc.isCanceled() { - return errRequestCanceled - } - return err + resp.Body = body + if rc.addedGzip && resp.Header.Get("Content-Encoding") == "gzip" { + resp.Body = &gzipReader{body: body} + resp.Header.Del("Content-Encoding") + resp.Header.Del("Content-Length") + resp.ContentLength = -1 + resp.Uncompressed = true } select { @@ -1182,13 +1496,16 @@ func (pc *persistConn) readLoop() { bodyEOF && !pc.sawEOF && pc.wroteRequest() && - tryPutIdleConn() + tryPutIdleConn(trace) if bodyEOF { eofc <- struct{}{} } case <-rc.req.Cancel: alive = false pc.t.CancelRequest(rc.req) + case <-rc.req.Context().Done(): + alive = false + pc.t.CancelRequest(rc.req) case <-pc.closech: alive = false } @@ -1197,15 +1514,6 @@ func (pc *persistConn) readLoop() { } } -func maybeUngzipResponse(resp *Response) { - if resp.Header.Get("Content-Encoding") == "gzip" { - resp.Header.Del("Content-Encoding") - resp.Header.Del("Content-Length") - resp.ContentLength = -1 - resp.Body = &gzipReader{body: resp.Body} - } -} - func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { if pc.closed != nil { return @@ -1224,19 +1532,29 @@ func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { // readResponse reads an HTTP response (or two, in the case of "Expect: // 100-continue") from the server. It returns the final non-100 one. -func (pc *persistConn) readResponse(rc requestAndChan) (resp *Response, err error) { +// trace is optional. +func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) { + if trace != nil && trace.GotFirstResponseByte != nil { + if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 { + trace.GotFirstResponseByte() + } + } resp, err = ReadResponse(pc.br, rc.req) if err != nil { return } if rc.continueCh != nil { if resp.StatusCode == 100 { + if trace != nil && trace.Got100Continue != nil { + trace.Got100Continue() + } rc.continueCh <- struct{}{} } else { close(rc.continueCh) } } if resp.StatusCode == 100 { + pc.readLimit = pc.maxHeaderResponseSize() // reset the limit resp, err = ReadResponse(pc.br, rc.req) if err != nil { return @@ -1268,24 +1586,33 @@ func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { } } +// nothingWrittenError wraps a write errors which ended up writing zero bytes. +type nothingWrittenError struct { + error +} + func (pc *persistConn) writeLoop() { + defer close(pc.writeLoopDone) for { select { case wr := <-pc.writech: - if pc.isBroken() { - wr.ch <- errors.New("http: can't write HTTP request on broken connection") - continue - } + startBytesWritten := pc.nwrite err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) if err == nil { err = pc.bw.Flush() } if err != nil { - pc.markBroken() wr.req.Request.closeBody() + if pc.nwrite == startBytesWritten { + err = nothingWrittenError{err} + } } pc.writeErrCh <- err // to the body reader, which might recycle us wr.ch <- err // to the roundTrip function + if err != nil { + pc.close(err) + return + } case <-pc.closech: return } @@ -1331,9 +1658,9 @@ type requestAndChan struct { req *Request ch chan responseAndError // unbuffered; always send in select on callerGone - // did the Transport (as opposed to the client code) add an - // Accept-Encoding gzip header? only if it we set it do - // we transparently decode the gzip. + // whether the Transport (as opposed to the user client code) + // added the Accept-Encoding gzip header. If the Transport + // set it, only then do we transparently decode the gzip. addedGzip bool // Optional blocking chan for Expect: 100-continue (for send). @@ -1353,7 +1680,7 @@ type writeRequest struct { req *transportRequest ch chan<- error - // Optional blocking chan for Expect: 100-continue (for recieve). + // Optional blocking chan for Expect: 100-continue (for receive). // If not nil, writeLoop blocks sending request body until // it receives from this chan. continueCh <-chan struct{} @@ -1369,7 +1696,6 @@ func (e *httpError) Timeout() bool { return e.timeout } func (e *httpError) Temporary() bool { return true } var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} -var errClosed error = &httpError{err: "net/http: server closed connection before response was received"} var errRequestCanceled = errors.New("net/http: request canceled") var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? @@ -1387,22 +1713,16 @@ var ( testHookReadLoopBeforeNextRead = nop ) -// beforeRespHeaderError is used to indicate when an IO error has occurred before -// any header data was received. -type beforeRespHeaderError struct { - error -} - func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { testHookEnterRoundTrip() if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) { pc.t.putOrCloseIdleConn(pc) return nil, errRequestCanceled } - pc.lk.Lock() + pc.mu.Lock() pc.numExpectedResponses++ headerFn := pc.mutateHeaderFunc - pc.lk.Unlock() + pc.mu.Unlock() if headerFn != nil { headerFn(req.extraHeaders()) @@ -1448,6 +1768,7 @@ func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err err // Write the request concurrently with waiting for a response, // in case the server decides to reply before reading our full // request body. + startBytesWritten := pc.nwrite writeErrCh := make(chan error, 1) pc.writech <- writeRequest{req, writeErrCh, continueCh} @@ -1472,7 +1793,7 @@ WaitResponse: if pc.isCanceled() { err = errRequestCanceled } - re = responseAndError{err: beforeRespHeaderError{err}} + re = responseAndError{err: err} pc.close(fmt.Errorf("write error: %v", err)) break WaitResponse } @@ -1482,26 +1803,21 @@ WaitResponse: respHeaderTimer = timer.C } case <-pc.closech: - var err error - if pc.isCanceled() { - err = errRequestCanceled - } else { - err = beforeRespHeaderError{fmt.Errorf("net/http: HTTP/1 transport connection broken: %v", pc.closed)} - } - re = responseAndError{err: err} + re = responseAndError{err: pc.mapRoundTripErrorAfterClosed(startBytesWritten)} break WaitResponse case <-respHeaderTimer: pc.close(errTimeout) re = responseAndError{err: errTimeout} break WaitResponse case re = <-resc: - if re.err != nil && pc.isCanceled() { - re.err = errRequestCanceled - } + re.err = pc.mapRoundTripErrorFromReadLoop(startBytesWritten, re.err) break WaitResponse case <-cancelChan: pc.t.CancelRequest(req.Request) cancelChan = nil + case <-req.Context().Done(): + pc.t.CancelRequest(req.Request) + cancelChan = nil } } @@ -1514,21 +1830,12 @@ WaitResponse: return re.res, re.err } -// markBroken marks a connection as broken (so it's not reused). -// It differs from close in that it doesn't close the underlying -// connection for use when it's still being read. -func (pc *persistConn) markBroken() { - pc.lk.Lock() - defer pc.lk.Unlock() - pc.broken = true -} - // markReused marks this connection as having been successfully used for a // request and response. func (pc *persistConn) markReused() { - pc.lk.Lock() + pc.mu.Lock() pc.reused = true - pc.lk.Unlock() + pc.mu.Unlock() } // close closes the underlying TCP connection and closes @@ -1537,8 +1844,8 @@ func (pc *persistConn) markReused() { // The provided err is only for testing and debugging; in normal // circumstances it should never be seen by users. func (pc *persistConn) close(err error) { - pc.lk.Lock() - defer pc.lk.Unlock() + pc.mu.Lock() + defer pc.mu.Unlock() pc.closeLocked(err) } @@ -1554,7 +1861,7 @@ func (pc *persistConn) closeLocked(err error) { // handlePendingDial's putOrCloseIdleConn when // it turns out the abandoned connection in // flight ended up negotiating an alternate - // protocol. We don't use the connection + // protocol. We don't use the connection // freelist for http2. That's done by the // alternate protocol's RoundTripper. } else { @@ -1579,7 +1886,11 @@ func canonicalAddr(url *url.URL) string { return addr } -// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most +// bodyEOFSignal is used by the HTTP/1 transport when reading response +// bodies to make sure we see the end of a response body before +// proceeding and reading on the connection again. +// +// It wraps a ReadCloser but runs fn (if non-nil) at most // once, right before its final (error-producing) Read or Close call // returns. fn should return the new error to return from Read or Close. // @@ -1595,12 +1906,14 @@ type bodyEOFSignal struct { earlyCloseFn func() error // optional alt Close func used if io.EOF not seen } +var errReadOnClosedResBody = errors.New("http: read on closed response body") + func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { es.mu.Lock() closed, rerr := es.closed, es.rerr es.mu.Unlock() if closed { - return 0, errors.New("http: read on closed response body") + return 0, errReadOnClosedResBody } if rerr != nil { return 0, rerr @@ -1645,16 +1958,29 @@ func (es *bodyEOFSignal) condfn(err error) error { // gzipReader wraps a response body so it can lazily // call gzip.NewReader on the first call to Read type gzipReader struct { - body io.ReadCloser // underlying Response.Body - zr io.Reader // lazily-initialized gzip reader + body *bodyEOFSignal // underlying HTTP/1 response body framing + zr *gzip.Reader // lazily-initialized gzip reader + zerr error // any error from gzip.NewReader; sticky } func (gz *gzipReader) Read(p []byte) (n int, err error) { if gz.zr == nil { - gz.zr, err = gzip.NewReader(gz.body) - if err != nil { - return 0, err + if gz.zerr == nil { + gz.zr, gz.zerr = gzip.NewReader(gz.body) } + if gz.zerr != nil { + return 0, gz.zerr + } + } + + gz.body.mu.Lock() + if gz.body.closed { + err = errReadOnClosedResBody + } + gz.body.mu.Unlock() + + if err != nil { + return 0, err } return gz.zr.Read(p) } @@ -1674,19 +2000,6 @@ func (tlsHandshakeTimeoutError) Timeout() bool { return true } func (tlsHandshakeTimeoutError) Temporary() bool { return true } func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } -type noteEOFReader struct { - r io.Reader - sawEOF *bool -} - -func (nr noteEOFReader) Read(p []byte) (n int, err error) { - n, err = nr.r.Read(p) - if err == io.EOF { - *nr.sawEOF = true - } - return -} - // fakeLocker is a sync.Locker which does nothing. It's used to guard // test-only fields when not under test, to avoid runtime atomic // overhead. @@ -1695,17 +2008,6 @@ type fakeLocker struct{} func (fakeLocker) Lock() {} func (fakeLocker) Unlock() {} -func isNetWriteError(err error) bool { - switch e := err.(type) { - case *url.Error: - return isNetWriteError(e.Err) - case *net.OpError: - return e.Op == "write" - default: - return false - } -} - // cloneTLSConfig returns a shallow clone of the exported // fields of cfg, ignoring the unexported sync.Once, which // contains a mutex and must not be copied. @@ -1722,25 +2024,27 @@ func cloneTLSConfig(cfg *tls.Config) *tls.Config { return &tls.Config{} } return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - SessionTicketsDisabled: cfg.SessionTicketsDisabled, - SessionTicketKey: cfg.SessionTicketKey, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + SessionTicketsDisabled: cfg.SessionTicketsDisabled, + SessionTicketKey: cfg.SessionTicketKey, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + DynamicRecordSizingDisabled: cfg.DynamicRecordSizingDisabled, + Renegotiation: cfg.Renegotiation, } } @@ -1753,22 +2057,63 @@ func cloneTLSClientConfig(cfg *tls.Config) *tls.Config { return &tls.Config{} } return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + DynamicRecordSizingDisabled: cfg.DynamicRecordSizingDisabled, + Renegotiation: cfg.Renegotiation, } } + +type connLRU struct { + ll *list.List // list.Element.Value type is of *persistConn + m map[*persistConn]*list.Element +} + +// add adds pc to the head of the linked list. +func (cl *connLRU) add(pc *persistConn) { + if cl.ll == nil { + cl.ll = list.New() + cl.m = make(map[*persistConn]*list.Element) + } + ele := cl.ll.PushFront(pc) + if _, ok := cl.m[pc]; ok { + panic("persistConn was already in LRU") + } + cl.m[pc] = ele +} + +func (cl *connLRU) removeOldest() *persistConn { + ele := cl.ll.Back() + pc := ele.Value.(*persistConn) + cl.ll.Remove(ele) + delete(cl.m, pc) + return pc +} + +// remove removes pc from cl. +func (cl *connLRU) remove(pc *persistConn) { + if ele, ok := cl.m[pc]; ok { + cl.ll.Remove(ele) + delete(cl.m, pc) + } +} + +// len returns the number of items in the cache. +func (cl *connLRU) len() int { + return len(cl.m) +} diff --git a/libgo/go/net/http/transport_internal_test.go b/libgo/go/net/http/transport_internal_test.go new file mode 100644 index 0000000..a157d90 --- /dev/null +++ b/libgo/go/net/http/transport_internal_test.go @@ -0,0 +1,69 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// White-box tests for transport.go (in package http instead of http_test). + +package http + +import ( + "errors" + "net" + "testing" +) + +// Issue 15446: incorrect wrapping of errors when server closes an idle connection. +func TestTransportPersistConnReadLoopEOF(t *testing.T) { + ln := newLocalListener(t) + defer ln.Close() + + connc := make(chan net.Conn, 1) + go func() { + defer close(connc) + c, err := ln.Accept() + if err != nil { + t.Error(err) + return + } + connc <- c + }() + + tr := new(Transport) + req, _ := NewRequest("GET", "http://"+ln.Addr().String(), nil) + treq := &transportRequest{Request: req} + cm := connectMethod{targetScheme: "http", targetAddr: ln.Addr().String()} + pc, err := tr.getConn(treq, cm) + if err != nil { + t.Fatal(err) + } + defer pc.close(errors.New("test over")) + + conn := <-connc + if conn == nil { + // Already called t.Error in the accept goroutine. + return + } + conn.Close() // simulate the server hanging up on the client + + _, err = pc.roundTrip(treq) + if err != errServerClosedConn && err != errServerClosedIdle { + t.Fatalf("roundTrip = %#v, %v; want errServerClosedConn or errServerClosedIdle", err, err) + } + + <-pc.closech + err = pc.closed + if err != errServerClosedConn && err != errServerClosedIdle { + t.Fatalf("pc.closed = %#v, %v; want errServerClosedConn or errServerClosedIdle", err, err) + } +} + +func newLocalListener(t *testing.T) net.Listener { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + ln, err = net.Listen("tcp6", "[::1]:0") + } + if err != nil { + t.Fatal(err) + } + return ln +} diff --git a/libgo/go/net/http/transport_test.go b/libgo/go/net/http/transport_test.go index 0c901b3..72b98f1 100644 --- a/libgo/go/net/http/transport_test.go +++ b/libgo/go/net/http/transport_test.go @@ -13,16 +13,20 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "crypto/rand" "crypto/tls" "errors" "fmt" + "internal/nettrace" + "internal/testenv" "io" "io/ioutil" "log" "net" . "net/http" "net/http/httptest" + "net/http/httptrace" "net/http/httputil" "net/http/internal" "net/url" @@ -379,8 +383,8 @@ func TestTransportMaxPerHostIdleConns(t *testing.T) { } })) defer ts.Close() - maxIdleConns := 2 - tr := &Transport{DisableKeepAlives: false, MaxIdleConnsPerHost: maxIdleConns} + maxIdleConnsPerHost := 2 + tr := &Transport{DisableKeepAlives: false, MaxIdleConnsPerHost: maxIdleConnsPerHost} c := &Client{Transport: tr} // Start 3 outstanding requests and wait for the server to get them. @@ -425,14 +429,65 @@ func TestTransportMaxPerHostIdleConns(t *testing.T) { resch <- "res2" <-donech - if e, g := 2, tr.IdleConnCountForTesting(cacheKey); e != g { - t.Errorf("after second response, expected %d idle conns; got %d", e, g) + if g, w := tr.IdleConnCountForTesting(cacheKey), 2; g != w { + t.Errorf("after second response, idle conns = %d; want %d", g, w) } resch <- "res3" <-donech - if e, g := maxIdleConns, tr.IdleConnCountForTesting(cacheKey); e != g { - t.Errorf("after third response, still expected %d idle conns; got %d", e, g) + if g, w := tr.IdleConnCountForTesting(cacheKey), maxIdleConnsPerHost; g != w { + t.Errorf("after third response, idle conns = %d; want %d", g, w) + } +} + +func TestTransportRemovesDeadIdleConnections(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see https://golang.org/issue/15464") + } + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + io.WriteString(w, r.RemoteAddr) + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + doReq := func(name string) string { + // Do a POST instead of a GET to prevent the Transport's + // idempotent request retry logic from kicking in... + res, err := c.Post(ts.URL, "", nil) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if res.StatusCode != 200 { + t.Fatalf("%s: %v", name, res.Status) + } + defer res.Body.Close() + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + return string(slurp) + } + + first := doReq("first") + keys1 := tr.IdleConnKeysForTesting() + + ts.CloseClientConnections() + + var keys2 []string + if !waitCondition(3*time.Second, 50*time.Millisecond, func() bool { + keys2 = tr.IdleConnKeysForTesting() + return len(keys2) == 0 + }) { + t.Fatalf("Transport didn't notice idle connection's death.\nbefore: %q\n after: %q\n", keys1, keys2) + } + + second := doReq("second") + if first == second { + t.Errorf("expected a different connection between requests. got %q both times", first) } } @@ -478,7 +533,7 @@ func TestTransportServerClosingUnexpectedly(t *testing.T) { // This test has an expected race. Sleeping for 25 ms prevents // it on most fast machines, causing the next fetch() call to - // succeed quickly. But if we do get errors, fetch() will retry 5 + // succeed quickly. But if we do get errors, fetch() will retry 5 // times with some delays between. time.Sleep(25 * time.Millisecond) @@ -518,7 +573,7 @@ func TestStressSurpriseServerCloses(t *testing.T) { // after each request completes, regardless of whether it failed. // If these are too high, OS X exhausts its ephemeral ports // and hangs waiting for them to transition TCP states. That's - // not what we want to test. TODO(bradfitz): use an io.Pipe + // not what we want to test. TODO(bradfitz): use an io.Pipe // dialer for this test instead? const ( numClients = 20 @@ -853,7 +908,7 @@ func TestTransportExpect100Continue(t *testing.T) { {path: "/100", body: []byte("hello"), sent: 5, status: 200}, // Got 100 followed by 200, entire body is sent. {path: "/200", body: []byte("hello"), sent: 0, status: 200}, // Got 200 without 100. body isn't sent. {path: "/500", body: []byte("hello"), sent: 0, status: 500}, // Got 500 without 100. body isn't sent. - {path: "/keepalive", body: []byte("hello"), sent: 0, status: 500}, // Althogh without Connection:close, body isn't sent. + {path: "/keepalive", body: []byte("hello"), sent: 0, status: 500}, // Although without Connection:close, body isn't sent. {path: "/timeout", body: []byte("hello"), sent: 5, status: 200}, // Timeout exceeded and entire body is sent. } @@ -923,7 +978,9 @@ func TestTransportGzipRecursive(t *testing.T) { })) defer ts.Close() - c := &Client{Transport: &Transport{}} + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) @@ -968,6 +1025,17 @@ func TestTransportGzipShort(t *testing.T) { } } +// Wait until number of goroutines is no greater than nmax, or time out. +func waitNumGoroutine(nmax int) int { + nfinal := runtime.NumGoroutine() + for ntries := 10; ntries > 0 && nfinal > nmax; ntries-- { + time.Sleep(50 * time.Millisecond) + runtime.GC() + nfinal = runtime.NumGoroutine() + } + return nfinal +} + // tests that persistent goroutine connections shut down when no longer desired. func TestTransportPersistConnLeak(t *testing.T) { setParallel(t) @@ -1019,14 +1087,11 @@ func TestTransportPersistConnLeak(t *testing.T) { } tr.CloseIdleConnections() - time.Sleep(100 * time.Millisecond) - runtime.GC() - runtime.GC() // even more. - nfinal := runtime.NumGoroutine() + nfinal := waitNumGoroutine(n0 + 5) growth := nfinal - n0 - // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. + // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. // Previously we were leaking one per numReq. if int(growth) > 5 { t.Logf("goroutine growth: %d -> %d -> %d (delta: %d)", n0, nhigh, nfinal, growth) @@ -1061,13 +1126,11 @@ func TestTransportPersistConnLeakShortBody(t *testing.T) { } nhigh := runtime.NumGoroutine() tr.CloseIdleConnections() - time.Sleep(400 * time.Millisecond) - runtime.GC() - nfinal := runtime.NumGoroutine() + nfinal := waitNumGoroutine(n0 + 5) growth := nfinal - n0 - // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. + // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. // Previously we were leaking one per numReq. t.Logf("goroutine growth: %d -> %d -> %d (delta: %d)", n0, nhigh, nfinal, growth) if int(growth) > 5 { @@ -1103,8 +1166,8 @@ func TestTransportIdleConnCrash(t *testing.T) { } // Test that the transport doesn't close the TCP connection early, -// before the response body has been read. This was a regression -// which sadly lacked a triggering test. The large response body made +// before the response body has been read. This was a regression +// which sadly lacked a triggering test. The large response body made // the old race easier to trigger. func TestIssue3644(t *testing.T) { defer afterTest(t) @@ -1199,7 +1262,7 @@ func TestTransportConcurrency(t *testing.T) { // Due to the Transport's "socket late binding" (see // idleConnCh in transport.go), the numReqs HTTP requests - // below can finish with a dial still outstanding. To keep + // below can finish with a dial still outstanding. To keep // the leak checker happy, keep track of pending dials and // wait for them to finish (and be closed or returned to the // idle pool) before we close idle connections. @@ -1617,7 +1680,13 @@ func TestCancelRequestWithChannel(t *testing.T) { } } -func TestCancelRequestWithChannelBeforeDo(t *testing.T) { +func TestCancelRequestWithChannelBeforeDo_Cancel(t *testing.T) { + testCancelRequestWithChannelBeforeDo(t, false) +} +func TestCancelRequestWithChannelBeforeDo_Context(t *testing.T) { + testCancelRequestWithChannelBeforeDo(t, true) +} +func testCancelRequestWithChannelBeforeDo(t *testing.T, withCtx bool) { setParallel(t) defer afterTest(t) unblockc := make(chan bool) @@ -1638,9 +1707,15 @@ func TestCancelRequestWithChannelBeforeDo(t *testing.T) { c := &Client{Transport: tr} req, _ := NewRequest("GET", ts.URL, nil) - ch := make(chan struct{}) - req.Cancel = ch - close(ch) + if withCtx { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + req = req.WithContext(ctx) + } else { + ch := make(chan struct{}) + req.Cancel = ch + close(ch) + } _, err := c.Do(req) if err == nil || !strings.Contains(err.Error(), "canceled") { @@ -1985,7 +2060,8 @@ type proxyFromEnvTest struct { env string // HTTP_PROXY httpsenv string // HTTPS_PROXY - noenv string // NO_RPXY + noenv string // NO_PROXY + reqmeth string // REQUEST_METHOD want string wanterr error @@ -2009,6 +2085,10 @@ func (t proxyFromEnvTest) String() string { space() fmt.Fprintf(&buf, "no_proxy=%q", t.noenv) } + if t.reqmeth != "" { + space() + fmt.Fprintf(&buf, "request_method=%q", t.reqmeth) + } req := "http://example.com" if t.req != "" { req = t.req @@ -2032,6 +2112,12 @@ var proxyFromEnvTests = []proxyFromEnvTest{ {req: "https://secure.tld/", env: "http.proxy.tld", httpsenv: "secure.proxy.tld", want: "http://secure.proxy.tld"}, {req: "https://secure.tld/", env: "http.proxy.tld", httpsenv: "https://secure.proxy.tld", want: "https://secure.proxy.tld"}, + // Issue 16405: don't use HTTP_PROXY in a CGI environment, + // where HTTP_PROXY can be attacker-controlled. + {env: "http://10.1.2.3:8080", reqmeth: "POST", + want: "<nil>", + wanterr: errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")}, + {want: "<nil>"}, {noenv: "example.com", req: "http://example.com/", env: "proxy", want: "<nil>"}, @@ -2047,6 +2133,7 @@ func TestProxyFromEnvironment(t *testing.T) { os.Setenv("HTTP_PROXY", tt.env) os.Setenv("HTTPS_PROXY", tt.httpsenv) os.Setenv("NO_PROXY", tt.noenv) + os.Setenv("REQUEST_METHOD", tt.reqmeth) ResetCachedEnvironment() reqURL := tt.req if reqURL == "" { @@ -2208,7 +2295,7 @@ func TestTransportTLSHandshakeTimeout(t *testing.T) { // Trying to repro golang.org/issue/3514 func TestTLSServerClosesConnection(t *testing.T) { defer afterTest(t) - setFlaky(t, 7634) + testenv.SkipFlaky(t, 7634) closedc := make(chan bool, 1) ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { @@ -2273,7 +2360,7 @@ func TestTLSServerClosesConnection(t *testing.T) { } // byteFromChanReader is an io.Reader that reads a single byte at a -// time from the channel. When the channel is closed, the reader +// time from the channel. When the channel is closed, the reader // returns io.EOF. type byteFromChanReader chan byte @@ -2405,7 +2492,7 @@ func (plan9SleepReader) Read(p []byte) (int, error) { // After the fix to unblock TCP Reads in // https://golang.org/cl/15941, this sleep is required // on plan9 to make sure TCP Writes before an - // immediate TCP close go out on the wire. On Plan 9, + // immediate TCP close go out on the wire. On Plan 9, // it seems that a hangup of a TCP connection with // queued data doesn't send the queued data first. // https://golang.org/issue/9554 @@ -2424,7 +2511,7 @@ func (f closerFunc) Close() error { return f() } // from (or finish writing to) the socket. // // NOTE: we resend a request only if the request is idempotent, we reused a -// keep-alive connection, and we haven't yet received any header data. This +// keep-alive connection, and we haven't yet received any header data. This // automatically prevents an infinite resend loop because we'll run out of the // cached keep-alive connections eventually. func TestRetryIdempotentRequestsOnError(t *testing.T) { @@ -2888,6 +2975,11 @@ func TestTransportAutomaticHTTP2(t *testing.T) { testTransportAutoHTTP(t, &Transport{}, true) } +// golang.org/issue/14391: also check DefaultTransport +func TestTransportAutomaticHTTP2_DefaultTransport(t *testing.T) { + testTransportAutoHTTP(t, DefaultTransport.(*Transport), true) +} + func TestTransportAutomaticHTTP2_TLSNextProto(t *testing.T) { testTransportAutoHTTP(t, &Transport{ TLSNextProto: make(map[string]func(string, *tls.Conn) RoundTripper), @@ -2903,6 +2995,21 @@ func TestTransportAutomaticHTTP2_TLSConfig(t *testing.T) { func TestTransportAutomaticHTTP2_ExpectContinueTimeout(t *testing.T) { testTransportAutoHTTP(t, &Transport{ ExpectContinueTimeout: 1 * time.Second, + }, true) +} + +func TestTransportAutomaticHTTP2_Dial(t *testing.T) { + var d net.Dialer + testTransportAutoHTTP(t, &Transport{ + Dial: d.Dial, + }, false) +} + +func TestTransportAutomaticHTTP2_DialTLS(t *testing.T) { + testTransportAutoHTTP(t, &Transport{ + DialTLS: func(network, addr string) (net.Conn, error) { + panic("unused") + }, }, false) } @@ -3033,6 +3140,377 @@ func TestNoCrashReturningTransportAltConn(t *testing.T) { <-handledPendingDial } +func TestTransportReuseConnection_Gzip_Chunked(t *testing.T) { + testTransportReuseConnection_Gzip(t, true) +} + +func TestTransportReuseConnection_Gzip_ContentLength(t *testing.T) { + testTransportReuseConnection_Gzip(t, false) +} + +// Make sure we re-use underlying TCP connection for gzipped responses too. +func testTransportReuseConnection_Gzip(t *testing.T, chunked bool) { + defer afterTest(t) + addr := make(chan string, 2) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + addr <- r.RemoteAddr + w.Header().Set("Content-Encoding", "gzip") + if chunked { + w.(Flusher).Flush() + } + w.Write(rgz) // arbitrary gzip response + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + for i := 0; i < 2; i++ { + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + buf := make([]byte, len(rgz)) + if n, err := io.ReadFull(res.Body, buf); err != nil { + t.Errorf("%d. ReadFull = %v, %v", i, n, err) + } + // Note: no res.Body.Close call. It should work without it, + // since the flate.Reader's internal buffering will hit EOF + // and that should be sufficient. + } + a1, a2 := <-addr, <-addr + if a1 != a2 { + t.Fatalf("didn't reuse connection") + } +} + +func TestTransportResponseHeaderLength(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.URL.Path == "/long" { + w.Header().Set("Long", strings.Repeat("a", 1<<20)) + } + })) + defer ts.Close() + + tr := &Transport{ + MaxResponseHeaderBytes: 512 << 10, + } + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + if res, err := c.Get(ts.URL); err != nil { + t.Fatal(err) + } else { + res.Body.Close() + } + + res, err := c.Get(ts.URL + "/long") + if err == nil { + defer res.Body.Close() + var n int64 + for k, vv := range res.Header { + for _, v := range vv { + n += int64(len(k)) + int64(len(v)) + } + } + t.Fatalf("Unexpected success. Got %v and %d bytes of response headers", res.Status, n) + } + if want := "server response headers exceeded 524288 bytes"; !strings.Contains(err.Error(), want) { + t.Errorf("got error: %v; want %q", err, want) + } +} + +func TestTransportEventTrace(t *testing.T) { testTransportEventTrace(t, h1Mode, false) } +func TestTransportEventTrace_h2(t *testing.T) { testTransportEventTrace(t, h2Mode, false) } + +// test a non-nil httptrace.ClientTrace but with all hooks set to zero. +func TestTransportEventTrace_NoHooks(t *testing.T) { testTransportEventTrace(t, h1Mode, true) } +func TestTransportEventTrace_NoHooks_h2(t *testing.T) { testTransportEventTrace(t, h2Mode, true) } + +func testTransportEventTrace(t *testing.T, h2 bool, noHooks bool) { + defer afterTest(t) + const resBody = "some body" + gotWroteReqEvent := make(chan struct{}) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + if _, err := ioutil.ReadAll(r.Body); err != nil { + t.Error(err) + } + if !noHooks { + select { + case <-gotWroteReqEvent: + case <-time.After(5 * time.Second): + t.Error("timeout waiting for WroteRequest event") + } + } + io.WriteString(w, resBody) + })) + defer cst.close() + + cst.tr.ExpectContinueTimeout = 1 * time.Second + + var mu sync.Mutex + var buf bytes.Buffer + logf := func(format string, args ...interface{}) { + mu.Lock() + defer mu.Unlock() + fmt.Fprintf(&buf, format, args...) + buf.WriteByte('\n') + } + + addrStr := cst.ts.Listener.Addr().String() + ip, port, err := net.SplitHostPort(addrStr) + if err != nil { + t.Fatal(err) + } + + // Install a fake DNS server. + ctx := context.WithValue(context.Background(), nettrace.LookupIPAltResolverKey{}, func(ctx context.Context, host string) ([]net.IPAddr, error) { + if host != "dns-is-faked.golang" { + t.Errorf("unexpected DNS host lookup for %q", host) + return nil, nil + } + return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil + }) + + req, _ := NewRequest("POST", cst.scheme()+"://dns-is-faked.golang:"+port, strings.NewReader("some body")) + trace := &httptrace.ClientTrace{ + GetConn: func(hostPort string) { logf("Getting conn for %v ...", hostPort) }, + GotConn: func(ci httptrace.GotConnInfo) { logf("got conn: %+v", ci) }, + GotFirstResponseByte: func() { logf("first response byte") }, + PutIdleConn: func(err error) { logf("PutIdleConn = %v", err) }, + DNSStart: func(e httptrace.DNSStartInfo) { logf("DNS start: %+v", e) }, + DNSDone: func(e httptrace.DNSDoneInfo) { logf("DNS done: %+v", e) }, + ConnectStart: func(network, addr string) { logf("ConnectStart: Connecting to %s %s ...", network, addr) }, + ConnectDone: func(network, addr string, err error) { + if err != nil { + t.Errorf("ConnectDone: %v", err) + } + logf("ConnectDone: connected to %s %s = %v", network, addr, err) + }, + Wait100Continue: func() { logf("Wait100Continue") }, + Got100Continue: func() { logf("Got100Continue") }, + WroteRequest: func(e httptrace.WroteRequestInfo) { + close(gotWroteReqEvent) + logf("WroteRequest: %+v", e) + }, + } + if noHooks { + // zero out all func pointers, trying to get some path to crash + *trace = httptrace.ClientTrace{} + } + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + + req.Header.Set("Expect", "100-continue") + res, err := cst.c.Do(req) + if err != nil { + t.Fatal(err) + } + logf("got roundtrip.response") + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + logf("consumed body") + if string(slurp) != resBody || res.StatusCode != 200 { + t.Fatalf("Got %q, %v; want %q, 200 OK", slurp, res.Status, resBody) + } + res.Body.Close() + + if noHooks { + // Done at this point. Just testing a full HTTP + // requests can happen with a trace pointing to a zero + // ClientTrace, full of nil func pointers. + return + } + + got := buf.String() + wantOnce := func(sub string) { + if strings.Count(got, sub) != 1 { + t.Errorf("expected substring %q exactly once in output.", sub) + } + } + wantOnceOrMore := func(sub string) { + if strings.Count(got, sub) == 0 { + t.Errorf("expected substring %q at least once in output.", sub) + } + } + wantOnce("Getting conn for dns-is-faked.golang:" + port) + wantOnce("DNS start: {Host:dns-is-faked.golang}") + wantOnce("DNS done: {Addrs:[{IP:" + ip + " Zone:}] Err:<nil> Coalesced:false}") + wantOnce("got conn: {") + wantOnceOrMore("Connecting to tcp " + addrStr) + wantOnceOrMore("connected to tcp " + addrStr + " = <nil>") + wantOnce("Reused:false WasIdle:false IdleTime:0s") + wantOnce("first response byte") + if !h2 { + wantOnce("PutIdleConn = <nil>") + } + wantOnce("Wait100Continue") + wantOnce("Got100Continue") + wantOnce("WroteRequest: {Err:<nil>}") + if strings.Contains(got, " to udp ") { + t.Errorf("should not see UDP (DNS) connections") + } + if t.Failed() { + t.Errorf("Output:\n%s", got) + } +} + +func TestTransportEventTraceRealDNS(t *testing.T) { + defer afterTest(t) + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + var mu sync.Mutex + var buf bytes.Buffer + logf := func(format string, args ...interface{}) { + mu.Lock() + defer mu.Unlock() + fmt.Fprintf(&buf, format, args...) + buf.WriteByte('\n') + } + + req, _ := NewRequest("GET", "http://dns-should-not-resolve.golang:80", nil) + trace := &httptrace.ClientTrace{ + DNSStart: func(e httptrace.DNSStartInfo) { logf("DNSStart: %+v", e) }, + DNSDone: func(e httptrace.DNSDoneInfo) { logf("DNSDone: %+v", e) }, + ConnectStart: func(network, addr string) { logf("ConnectStart: %s %s", network, addr) }, + ConnectDone: func(network, addr string, err error) { logf("ConnectDone: %s %s %v", network, addr, err) }, + } + req = req.WithContext(httptrace.WithClientTrace(context.Background(), trace)) + + resp, err := c.Do(req) + if err == nil { + resp.Body.Close() + t.Fatal("expected error during DNS lookup") + } + + got := buf.String() + wantSub := func(sub string) { + if !strings.Contains(got, sub) { + t.Errorf("expected substring %q in output.", sub) + } + } + wantSub("DNSStart: {Host:dns-should-not-resolve.golang}") + wantSub("DNSDone: {Addrs:[] Err:") + if strings.Contains(got, "ConnectStart") || strings.Contains(got, "ConnectDone") { + t.Errorf("should not see Connect events") + } + if t.Failed() { + t.Errorf("Output:\n%s", got) + } +} + +func TestTransportMaxIdleConns(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + // No body for convenience. + })) + defer ts.Close() + tr := &Transport{ + MaxIdleConns: 4, + } + defer tr.CloseIdleConnections() + + ip, port, err := net.SplitHostPort(ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + c := &Client{Transport: tr} + ctx := context.WithValue(context.Background(), nettrace.LookupIPAltResolverKey{}, func(ctx context.Context, host string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil + }) + + hitHost := func(n int) { + req, _ := NewRequest("GET", fmt.Sprintf("http://host-%d.dns-is-faked.golang:"+port, n), nil) + req = req.WithContext(ctx) + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + } + for i := 0; i < 4; i++ { + hitHost(i) + } + want := []string{ + "|http|host-0.dns-is-faked.golang:" + port, + "|http|host-1.dns-is-faked.golang:" + port, + "|http|host-2.dns-is-faked.golang:" + port, + "|http|host-3.dns-is-faked.golang:" + port, + } + if got := tr.IdleConnKeysForTesting(); !reflect.DeepEqual(got, want) { + t.Fatalf("idle conn keys mismatch.\n got: %q\nwant: %q\n", got, want) + } + + // Now hitting the 5th host should kick out the first host: + hitHost(4) + want = []string{ + "|http|host-1.dns-is-faked.golang:" + port, + "|http|host-2.dns-is-faked.golang:" + port, + "|http|host-3.dns-is-faked.golang:" + port, + "|http|host-4.dns-is-faked.golang:" + port, + } + if got := tr.IdleConnKeysForTesting(); !reflect.DeepEqual(got, want) { + t.Fatalf("idle conn keys mismatch after 5th host.\n got: %q\nwant: %q\n", got, want) + } +} + +func TestTransportIdleConnTimeout(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + defer afterTest(t) + + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + // No body for convenience. + })) + defer ts.Close() + + const timeout = 1 * time.Second + tr := &Transport{ + IdleConnTimeout: timeout, + } + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + var conn string + doReq := func(n int) { + req, _ := NewRequest("GET", ts.URL, nil) + req = req.WithContext(httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ + PutIdleConn: func(err error) { + if err != nil { + t.Errorf("failed to keep idle conn: %v", err) + } + }, + })) + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + conns := tr.IdleConnStrsForTesting() + if len(conns) != 1 { + t.Fatalf("req %v: unexpected number of idle conns: %q", n, conns) + } + if conn == "" { + conn = conns[0] + } + if conn != conns[0] { + t.Fatalf("req %v: cached connection changed; expected the same one throughout the test", n) + } + } + for i := 0; i < 3; i++ { + doReq(i) + time.Sleep(timeout / 2) + } + time.Sleep(timeout * 3 / 2) + if got := tr.IdleConnStrsForTesting(); len(got) != 0 { + t.Errorf("idle conns = %q; want none", got) + } +} + var errFakeRoundTrip = errors.New("fake roundtrip") type funcRoundTripper func() diff --git a/libgo/go/net/interface.go b/libgo/go/net/interface.go index 9c7b5da..52b857c 100644 --- a/libgo/go/net/interface.go +++ b/libgo/go/net/interface.go @@ -1,10 +1,14 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net -import "errors" +import ( + "errors" + "sync" + "time" +) var ( errInvalidInterface = errors.New("invalid network interface") @@ -15,7 +19,7 @@ var ( ) // Interface represents a mapping between network interface name -// and index. It also represents network interface facility +// and index. It also represents network interface facility // information. type Interface struct { Index int // positive integer that starts at one, zero is never used @@ -88,9 +92,12 @@ func (ifi *Interface) MulticastAddrs() ([]Addr, error) { func Interfaces() ([]Interface, error) { ift, err := interfaceTable(0) if err != nil { - err = &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: err} + return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: err} } - return ift, err + if len(ift) != 0 { + zoneCache.update(ift) + } + return ift, nil } // InterfaceAddrs returns a list of the system's network interface @@ -137,6 +144,9 @@ func InterfaceByName(name string) (*Interface, error) { if err != nil { return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: err} } + if len(ift) != 0 { + zoneCache.update(ift) + } for _, ifi := range ift { if name == ifi.Name { return &ifi, nil @@ -144,3 +154,68 @@ func InterfaceByName(name string) (*Interface, error) { } return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: errNoSuchInterface} } + +// An ipv6ZoneCache represents a cache holding partial network +// interface information. It is used for reducing the cost of IPv6 +// addressing scope zone resolution. +type ipv6ZoneCache struct { + sync.RWMutex // guard the following + lastFetched time.Time // last time routing information was fetched + toIndex map[string]int // interface name to its index + toName map[int]string // interface index to its name +} + +var zoneCache = ipv6ZoneCache{ + toIndex: make(map[string]int), + toName: make(map[int]string), +} + +func (zc *ipv6ZoneCache) update(ift []Interface) { + zc.Lock() + defer zc.Unlock() + now := time.Now() + if zc.lastFetched.After(now.Add(-60 * time.Second)) { + return + } + zc.lastFetched = now + if len(ift) == 0 { + var err error + if ift, err = interfaceTable(0); err != nil { + return + } + } + zc.toIndex = make(map[string]int, len(ift)) + zc.toName = make(map[int]string, len(ift)) + for _, ifi := range ift { + zc.toIndex[ifi.Name] = ifi.Index + zc.toName[ifi.Index] = ifi.Name + } +} + +func zoneToString(zone int) string { + if zone == 0 { + return "" + } + zoneCache.update(nil) + zoneCache.RLock() + defer zoneCache.RUnlock() + name, ok := zoneCache.toName[zone] + if !ok { + name = uitoa(uint(zone)) + } + return name +} + +func zoneToInt(zone string) int { + if zone == "" { + return 0 + } + zoneCache.update(nil) + zoneCache.RLock() + defer zoneCache.RUnlock() + index, ok := zoneCache.toIndex[zone] + if !ok { + index, _, _ = dtoi(zone, 0) + } + return index +} diff --git a/libgo/go/net/interface_bsd.go b/libgo/go/net/interface_bsd.go index 208f37f..35b1c26 100644 --- a/libgo/go/net/interface_bsd.go +++ b/libgo/go/net/interface_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,74 +7,54 @@ package net import ( - "os" "syscall" - "unsafe" + + "golang_org/x/net/route" ) // If the ifindex is zero, interfaceTable returns mappings of all -// network interfaces. Otherwise it returns a mapping of a specific +// network interfaces. Otherwise it returns a mapping of a specific // interface. func interfaceTable(ifindex int) ([]Interface, error) { - tab, err := syscall.RouteRIB(syscall.NET_RT_IFLIST, ifindex) + msgs, err := interfaceMessages(ifindex) if err != nil { - return nil, os.NewSyscallError("routerib", err) + return nil, err } - msgs, err := syscall.ParseRoutingMessage(tab) - if err != nil { - return nil, os.NewSyscallError("parseroutingmessage", err) + n := len(msgs) + if ifindex != 0 { + n = 1 } - return parseInterfaceTable(ifindex, msgs) -} - -func parseInterfaceTable(ifindex int, msgs []syscall.RoutingMessage) ([]Interface, error) { - var ift []Interface -loop: + ift := make([]Interface, n) + n = 0 for _, m := range msgs { switch m := m.(type) { - case *syscall.InterfaceMessage: - if ifindex == 0 || ifindex == int(m.Header.Index) { - ifi, err := newLink(m) - if err != nil { - return nil, err - } - ift = append(ift, *ifi) - if ifindex == int(m.Header.Index) { - break loop + case *route.InterfaceMessage: + if ifindex != 0 && ifindex != m.Index { + continue + } + ift[n].Index = m.Index + ift[n].Name = m.Name + ift[n].Flags = linkFlags(m.Flags) + if sa, ok := m.Addrs[syscall.RTAX_IFP].(*route.LinkAddr); ok && len(sa.Addr) > 0 { + ift[n].HardwareAddr = make([]byte, len(sa.Addr)) + copy(ift[n].HardwareAddr, sa.Addr) + } + for _, sys := range m.Sys() { + if imx, ok := sys.(*route.InterfaceMetrics); ok { + ift[n].MTU = imx.MTU + break } } + n++ + if ifindex == m.Index { + return ift[:n], nil + } } } - return ift, nil -} - -func newLink(m *syscall.InterfaceMessage) (*Interface, error) { - sas, err := syscall.ParseRoutingSockaddr(m) - if err != nil { - return nil, os.NewSyscallError("parseroutingsockaddr", err) - } - ifi := &Interface{Index: int(m.Header.Index), Flags: linkFlags(m.Header.Flags)} - sa, _ := sas[syscall.RTAX_IFP].(*syscall.SockaddrDatalink) - if sa != nil { - // NOTE: SockaddrDatalink.Data is minimum work area, - // can be larger. - m.Data = m.Data[unsafe.Offsetof(sa.Data):] - var name [syscall.IFNAMSIZ]byte - for i := 0; i < int(sa.Nlen); i++ { - name[i] = byte(m.Data[i]) - } - ifi.Name = string(name[:sa.Nlen]) - ifi.MTU = int(m.Header.Data.Mtu) - addr := make([]byte, sa.Alen) - for i := 0; i < int(sa.Alen); i++ { - addr[i] = byte(m.Data[int(sa.Nlen)+i]) - } - ifi.HardwareAddr = addr[:sa.Alen] - } - return ifi, nil + return ift[:n], nil } -func linkFlags(rawFlags int32) Flags { +func linkFlags(rawFlags int) Flags { var f Flags if rawFlags&syscall.IFF_UP != 0 { f |= FlagUp @@ -95,81 +75,44 @@ func linkFlags(rawFlags int32) Flags { } // If the ifi is nil, interfaceAddrTable returns addresses for all -// network interfaces. Otherwise it returns addresses for a specific +// network interfaces. Otherwise it returns addresses for a specific // interface. func interfaceAddrTable(ifi *Interface) ([]Addr, error) { index := 0 if ifi != nil { index = ifi.Index } - tab, err := syscall.RouteRIB(syscall.NET_RT_IFLIST, index) - if err != nil { - return nil, os.NewSyscallError("routerib", err) - } - msgs, err := syscall.ParseRoutingMessage(tab) + msgs, err := interfaceMessages(index) if err != nil { - return nil, os.NewSyscallError("parseroutingmessage", err) + return nil, err } - var ift []Interface - if index == 0 { - ift, err = parseInterfaceTable(index, msgs) - if err != nil { - return nil, err - } - } - var ifat []Addr + ifat := make([]Addr, 0, len(msgs)) for _, m := range msgs { switch m := m.(type) { - case *syscall.InterfaceAddrMessage: - if index == 0 || index == int(m.Header.Index) { - if index == 0 { - var err error - ifi, err = interfaceByIndex(ift, int(m.Header.Index)) - if err != nil { - return nil, err - } - } - ifa, err := newAddr(ifi, m) - if err != nil { - return nil, err - } - if ifa != nil { - ifat = append(ifat, ifa) - } + case *route.InterfaceAddrMessage: + if index != 0 && index != m.Index { + continue + } + var mask IPMask + switch sa := m.Addrs[syscall.RTAX_NETMASK].(type) { + case *route.Inet4Addr: + mask = IPv4Mask(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) + case *route.Inet6Addr: + mask = make(IPMask, IPv6len) + copy(mask, sa.IP[:]) + } + var ip IP + switch sa := m.Addrs[syscall.RTAX_IFA].(type) { + case *route.Inet4Addr: + ip = IPv4(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) + case *route.Inet6Addr: + ip = make(IP, IPv6len) + copy(ip, sa.IP[:]) + } + if ip != nil && mask != nil { // NetBSD may contain route.LinkAddr + ifat = append(ifat, &IPNet{IP: ip, Mask: mask}) } } } return ifat, nil } - -func newAddr(ifi *Interface, m *syscall.InterfaceAddrMessage) (*IPNet, error) { - sas, err := syscall.ParseRoutingSockaddr(m) - if err != nil { - return nil, os.NewSyscallError("parseroutingsockaddr", err) - } - ifa := &IPNet{} - switch sa := sas[syscall.RTAX_NETMASK].(type) { - case *syscall.SockaddrInet4: - ifa.Mask = IPv4Mask(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3]) - case *syscall.SockaddrInet6: - ifa.Mask = make(IPMask, IPv6len) - copy(ifa.Mask, sa.Addr[:]) - } - switch sa := sas[syscall.RTAX_IFA].(type) { - case *syscall.SockaddrInet4: - ifa.IP = IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3]) - case *syscall.SockaddrInet6: - ifa.IP = make(IP, IPv6len) - copy(ifa.IP, sa.Addr[:]) - // NOTE: KAME based IPv6 protcol stack usually embeds - // the interface index in the interface-local or - // link-local address as the kernel-internal form. - if ifa.IP.IsLinkLocalUnicast() { - ifa.IP[2], ifa.IP[3] = 0, 0 - } - } - if ifa.IP == nil || ifa.Mask == nil { - return nil, nil // Sockaddrs contain syscall.SockaddrDatalink on NetBSD - } - return ifa, nil -} diff --git a/libgo/go/net/interface_bsdvar.go b/libgo/go/net/interface_bsdvar.go new file mode 100644 index 0000000..0b84ca3 --- /dev/null +++ b/libgo/go/net/interface_bsdvar.go @@ -0,0 +1,28 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build dragonfly netbsd openbsd + +package net + +import ( + "syscall" + + "golang_org/x/net/route" +) + +func interfaceMessages(ifindex int) ([]route.Message, error) { + rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFLIST, ifindex) + if err != nil { + return nil, err + } + return route.ParseRIB(syscall.NET_RT_IFLIST, rib) +} + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + // TODO(mikio): Implement this like other platforms. + return nil, nil +} diff --git a/libgo/go/net/interface_darwin.go b/libgo/go/net/interface_darwin.go index b7a3338..2ec8e1c 100644 --- a/libgo/go/net/interface_darwin.go +++ b/libgo/go/net/interface_darwin.go @@ -1,62 +1,53 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( - "os" "syscall" + + "golang_org/x/net/route" ) +func interfaceMessages(ifindex int) ([]route.Message, error) { + rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFLIST, ifindex) + if err != nil { + return nil, err + } + return route.ParseRIB(syscall.NET_RT_IFLIST, rib) +} + // interfaceMulticastAddrTable returns addresses for a specific // interface. func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - tab, err := syscall.RouteRIB(syscall.NET_RT_IFLIST2, ifi.Index) + rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFLIST2, ifi.Index) if err != nil { - return nil, os.NewSyscallError("routerib", err) + return nil, err } - msgs, err := syscall.ParseRoutingMessage(tab) + msgs, err := route.ParseRIB(syscall.NET_RT_IFLIST2, rib) if err != nil { - return nil, os.NewSyscallError("parseroutingmessage", err) + return nil, err } - var ifmat []Addr + ifmat := make([]Addr, 0, len(msgs)) for _, m := range msgs { switch m := m.(type) { - case *syscall.InterfaceMulticastAddrMessage: - if ifi.Index == int(m.Header.Index) { - ifma, err := newMulticastAddr(ifi, m) - if err != nil { - return nil, err - } - if ifma != nil { - ifmat = append(ifmat, ifma) - } + case *route.InterfaceMulticastAddrMessage: + if ifi.Index != m.Index { + continue + } + var ip IP + switch sa := m.Addrs[syscall.RTAX_IFA].(type) { + case *route.Inet4Addr: + ip = IPv4(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) + case *route.Inet6Addr: + ip = make(IP, IPv6len) + copy(ip, sa.IP[:]) + } + if ip != nil { + ifmat = append(ifmat, &IPAddr{IP: ip}) } } } return ifmat, nil } - -func newMulticastAddr(ifi *Interface, m *syscall.InterfaceMulticastAddrMessage) (*IPAddr, error) { - sas, err := syscall.ParseRoutingSockaddr(m) - if err != nil { - return nil, os.NewSyscallError("parseroutingsockaddr", err) - } - switch sa := sas[syscall.RTAX_IFA].(type) { - case *syscall.SockaddrInet4: - return &IPAddr{IP: IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3])}, nil - case *syscall.SockaddrInet6: - ifma := IPAddr{IP: make(IP, IPv6len)} - copy(ifma.IP, sa.Addr[:]) - // NOTE: KAME based IPv6 protcol stack usually embeds - // the interface index in the interface-local or - // link-local address as the kernel-internal form. - if ifma.IP.IsInterfaceLocalMulticast() || ifma.IP.IsLinkLocalMulticast() { - ifma.IP[2], ifma.IP[3] = 0, 0 - } - return &ifma, nil - default: - return nil, nil - } -} diff --git a/libgo/go/net/interface_dragonfly.go b/libgo/go/net/interface_dragonfly.go deleted file mode 100644 index c9ce5a7..0000000 --- a/libgo/go/net/interface_dragonfly.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -// interfaceMulticastAddrTable returns addresses for a specific -// interface. -func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - // TODO(mikio): Implement this like other platforms. - return nil, nil -} diff --git a/libgo/go/net/interface_freebsd.go b/libgo/go/net/interface_freebsd.go index c42d90b..8a7d6f6 100644 --- a/libgo/go/net/interface_freebsd.go +++ b/libgo/go/net/interface_freebsd.go @@ -1,62 +1,58 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( - "os" "syscall" + + "golang_org/x/net/route" ) +func interfaceMessages(ifindex int) ([]route.Message, error) { + typ := route.RIBType(syscall.NET_RT_IFLISTL) + rib, err := route.FetchRIB(syscall.AF_UNSPEC, typ, ifindex) + if err != nil { + typ = route.RIBType(syscall.NET_RT_IFLIST) + rib, err = route.FetchRIB(syscall.AF_UNSPEC, typ, ifindex) + } + if err != nil { + return nil, err + } + return route.ParseRIB(typ, rib) +} + // interfaceMulticastAddrTable returns addresses for a specific // interface. func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - tab, err := syscall.RouteRIB(syscall.NET_RT_IFMALIST, ifi.Index) + rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFMALIST, ifi.Index) if err != nil { - return nil, os.NewSyscallError("routerib", err) + return nil, err } - msgs, err := syscall.ParseRoutingMessage(tab) + msgs, err := route.ParseRIB(syscall.NET_RT_IFMALIST, rib) if err != nil { - return nil, os.NewSyscallError("parseroutingmessage", err) + return nil, err } - var ifmat []Addr + ifmat := make([]Addr, 0, len(msgs)) for _, m := range msgs { switch m := m.(type) { - case *syscall.InterfaceMulticastAddrMessage: - if ifi.Index == int(m.Header.Index) { - ifma, err := newMulticastAddr(ifi, m) - if err != nil { - return nil, err - } - if ifma != nil { - ifmat = append(ifmat, ifma) - } + case *route.InterfaceMulticastAddrMessage: + if ifi.Index != m.Index { + continue + } + var ip IP + switch sa := m.Addrs[syscall.RTAX_IFA].(type) { + case *route.Inet4Addr: + ip = IPv4(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) + case *route.Inet6Addr: + ip = make(IP, IPv6len) + copy(ip, sa.IP[:]) + } + if ip != nil { + ifmat = append(ifmat, &IPAddr{IP: ip}) } } } return ifmat, nil } - -func newMulticastAddr(ifi *Interface, m *syscall.InterfaceMulticastAddrMessage) (*IPAddr, error) { - sas, err := syscall.ParseRoutingSockaddr(m) - if err != nil { - return nil, os.NewSyscallError("parseroutingsockaddr", err) - } - switch sa := sas[syscall.RTAX_IFA].(type) { - case *syscall.SockaddrInet4: - return &IPAddr{IP: IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3])}, nil - case *syscall.SockaddrInet6: - ifma := IPAddr{IP: make(IP, IPv6len)} - copy(ifma.IP, sa.Addr[:]) - // NOTE: KAME based IPv6 protcol stack usually embeds - // the interface index in the interface-local or - // link-local address as the kernel-internal form. - if ifma.IP.IsInterfaceLocalMulticast() || ifma.IP.IsLinkLocalMulticast() { - ifma.IP[2], ifma.IP[3] = 0, 0 - } - return &ifma, nil - default: - return nil, nil - } -} diff --git a/libgo/go/net/interface_linux.go b/libgo/go/net/interface_linux.go index ef20429..5e391b2 100644 --- a/libgo/go/net/interface_linux.go +++ b/libgo/go/net/interface_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -11,7 +11,7 @@ import ( ) // If the ifindex is zero, interfaceTable returns mappings of all -// network interfaces. Otherwise it returns a mapping of a specific +// network interfaces. Otherwise it returns a mapping of a specific // interface. func interfaceTable(ifindex int) ([]Interface, error) { tab, err := syscall.NetlinkRIB(syscall.RTM_GETLINK, syscall.AF_UNSPEC) @@ -115,7 +115,7 @@ func linkFlags(rawFlags uint32) Flags { } // If the ifi is nil, interfaceAddrTable returns addresses for all -// network interfaces. Otherwise it returns addresses for a specific +// network interfaces. Otherwise it returns addresses for a specific // interface. func interfaceAddrTable(ifi *Interface) ([]Addr, error) { tab, err := syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_UNSPEC) diff --git a/libgo/go/net/interface_netbsd.go b/libgo/go/net/interface_netbsd.go deleted file mode 100644 index c9ce5a7..0000000 --- a/libgo/go/net/interface_netbsd.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -// interfaceMulticastAddrTable returns addresses for a specific -// interface. -func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - // TODO(mikio): Implement this like other platforms. - return nil, nil -} diff --git a/libgo/go/net/interface_openbsd.go b/libgo/go/net/interface_openbsd.go deleted file mode 100644 index c9ce5a7..0000000 --- a/libgo/go/net/interface_openbsd.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -// interfaceMulticastAddrTable returns addresses for a specific -// interface. -func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - // TODO(mikio): Implement this like other platforms. - return nil, nil -} diff --git a/libgo/go/net/interface_stub.go b/libgo/go/net/interface_stub.go index c38fb7f..f64174c 100644 --- a/libgo/go/net/interface_stub.go +++ b/libgo/go/net/interface_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,14 +7,14 @@ package net // If the ifindex is zero, interfaceTable returns mappings of all -// network interfaces. Otherwise it returns a mapping of a specific +// network interfaces. Otherwise it returns a mapping of a specific // interface. func interfaceTable(ifindex int) ([]Interface, error) { return nil, nil } // If the ifi is nil, interfaceAddrTable returns addresses for all -// network interfaces. Otherwise it returns addresses for a specific +// network interfaces. Otherwise it returns addresses for a specific // interface. func interfaceAddrTable(ifi *Interface) ([]Addr, error) { return nil, nil diff --git a/libgo/go/net/interface_test.go b/libgo/go/net/interface_test.go index 7bdd924..4c695b9 100644 --- a/libgo/go/net/interface_test.go +++ b/libgo/go/net/interface_test.go @@ -1,17 +1,18 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "fmt" "reflect" "runtime" "testing" ) // loopbackInterface returns an available logical network interface -// for loopback tests. It returns nil if no suitable interface is +// for loopback tests. It returns nil if no suitable interface is // found. func loopbackInterface() *Interface { ift, err := Interfaces() @@ -47,20 +48,11 @@ func ipv6LinkLocalUnicastAddr(ifi *Interface) string { return "" } -type routeStats struct { - loop int // # of active loopback interfaces - other int // # of active other interfaces - - uni4, uni6 int // # of active connected unicast, anycast routes - multi4, multi6 int // # of active connected multicast route clones -} - func TestInterfaces(t *testing.T) { ift, err := Interfaces() if err != nil { t.Fatal(err) } - var stats routeStats for _, ifi := range ift { ifxi, err := InterfaceByIndex(ifi.Index) if err != nil { @@ -76,56 +68,7 @@ func TestInterfaces(t *testing.T) { if !reflect.DeepEqual(ifxn, &ifi) { t.Errorf("got %v; want %v", ifxn, ifi) } - t.Logf("%q: flags %q, ifindex %v, mtu %v", ifi.Name, ifi.Flags.String(), ifi.Index, ifi.MTU) - t.Logf("hardware address %q", ifi.HardwareAddr.String()) - if ifi.Flags&FlagUp != 0 { - if ifi.Flags&FlagLoopback != 0 { - stats.loop++ - } else { - stats.other++ - } - } - n4, n6 := testInterfaceAddrs(t, &ifi) - stats.uni4 += n4 - stats.uni6 += n6 - n4, n6 = testInterfaceMulticastAddrs(t, &ifi) - stats.multi4 += n4 - stats.multi6 += n6 - } - switch runtime.GOOS { - case "nacl", "plan9", "solaris": - default: - // Test the existence of connected unicast routes for - // IPv4. - if supportsIPv4 && stats.loop+stats.other > 0 && stats.uni4 == 0 { - t.Errorf("num IPv4 unicast routes = 0; want >0; summary: %+v", stats) - } - // Test the existence of connected unicast routes for - // IPv6. We can assume the existence of ::1/128 when - // at least one looopback interface is installed. - if supportsIPv6 && stats.loop > 0 && stats.uni6 == 0 { - t.Errorf("num IPv6 unicast routes = 0; want >0; summary: %+v", stats) - } - } - switch runtime.GOOS { - case "dragonfly", "nacl", "netbsd", "openbsd", "plan9", "solaris": - default: - // Test the existence of connected multicast route - // clones for IPv4. Unlike IPv6, IPv4 multicast - // capability is not a mandatory feature, and so this - // test is disabled. - //if supportsIPv4 && stats.loop > 0 && stats.uni4 > 1 && stats.multi4 == 0 { - // t.Errorf("num IPv4 multicast route clones = 0; want >0; summary: %+v", stats) - //} - // Test the existence of connected multicast route - // clones for IPv6. Some platform never uses loopback - // interface as the nexthop for multicast routing. - // We can assume the existence of connected multicast - // route clones when at least two connected unicast - // routes, ::1/128 and other, are installed. - if supportsIPv6 && stats.loop > 0 && stats.uni6 > 1 && stats.multi6 == 0 { - t.Errorf("num IPv6 multicast route clones = 0; want >0; summary: %+v", stats) - } + t.Logf("%s: flags=%v index=%d mtu=%d hwaddr=%v", ifi.Name, ifi.Flags, ifi.Index, ifi.MTU, ifi.HardwareAddr) } } @@ -134,132 +77,217 @@ func TestInterfaceAddrs(t *testing.T) { if err != nil { t.Fatal(err) } - var stats routeStats - for _, ifi := range ift { - if ifi.Flags&FlagUp != 0 { - if ifi.Flags&FlagLoopback != 0 { - stats.loop++ - } else { - stats.other++ - } - } - } + ifStats := interfaceStats(ift) ifat, err := InterfaceAddrs() if err != nil { t.Fatal(err) } - stats.uni4, stats.uni6 = testAddrs(t, ifat) - // Test the existence of connected unicast routes for IPv4. - if supportsIPv4 && stats.loop+stats.other > 0 && stats.uni4 == 0 { - t.Errorf("num IPv4 unicast routes = 0; want >0; summary: %+v", stats) + uniStats, err := validateInterfaceUnicastAddrs(ifat) + if err != nil { + t.Fatal(err) } - // Test the existence of connected unicast routes for IPv6. - // We can assume the existence of ::1/128 when at least one - // looopback interface is installed. - if supportsIPv6 && stats.loop > 0 && stats.uni6 == 0 { - t.Errorf("num IPv6 unicast routes = 0; want >0; summary: %+v", stats) + if err := checkUnicastStats(ifStats, uniStats); err != nil { + t.Fatal(err) } } -func testInterfaceAddrs(t *testing.T, ifi *Interface) (naf4, naf6 int) { - ifat, err := ifi.Addrs() +func TestInterfaceUnicastAddrs(t *testing.T) { + ift, err := Interfaces() + if err != nil { + t.Fatal(err) + } + ifStats := interfaceStats(ift) if err != nil { t.Fatal(err) } - return testAddrs(t, ifat) + var uniStats routeStats + for _, ifi := range ift { + ifat, err := ifi.Addrs() + if err != nil { + t.Fatal(ifi, err) + } + stats, err := validateInterfaceUnicastAddrs(ifat) + if err != nil { + t.Fatal(ifi, err) + } + uniStats.ipv4 += stats.ipv4 + uniStats.ipv6 += stats.ipv6 + } + if err := checkUnicastStats(ifStats, &uniStats); err != nil { + t.Fatal(err) + } } -func testInterfaceMulticastAddrs(t *testing.T, ifi *Interface) (nmaf4, nmaf6 int) { - ifmat, err := ifi.MulticastAddrs() +func TestInterfaceMulticastAddrs(t *testing.T) { + ift, err := Interfaces() + if err != nil { + t.Fatal(err) + } + ifStats := interfaceStats(ift) + ifat, err := InterfaceAddrs() + if err != nil { + t.Fatal(err) + } + uniStats, err := validateInterfaceUnicastAddrs(ifat) if err != nil { t.Fatal(err) } - return testMulticastAddrs(t, ifmat) + var multiStats routeStats + for _, ifi := range ift { + ifmat, err := ifi.MulticastAddrs() + if err != nil { + t.Fatal(ifi, err) + } + stats, err := validateInterfaceMulticastAddrs(ifmat) + if err != nil { + t.Fatal(ifi, err) + } + multiStats.ipv4 += stats.ipv4 + multiStats.ipv6 += stats.ipv6 + } + if err := checkMulticastStats(ifStats, uniStats, &multiStats); err != nil { + t.Fatal(err) + } } -func testAddrs(t *testing.T, ifat []Addr) (naf4, naf6 int) { +type ifStats struct { + loop int // # of active loopback interfaces + other int // # of active other interfaces +} + +func interfaceStats(ift []Interface) *ifStats { + var stats ifStats + for _, ifi := range ift { + if ifi.Flags&FlagUp != 0 { + if ifi.Flags&FlagLoopback != 0 { + stats.loop++ + } else { + stats.other++ + } + } + } + return &stats +} + +type routeStats struct { + ipv4, ipv6 int // # of active connected unicast, anycast or multicast routes +} + +func validateInterfaceUnicastAddrs(ifat []Addr) (*routeStats, error) { + // Note: BSD variants allow assigning any IPv4/IPv6 address + // prefix to IP interface. For example, + // - 0.0.0.0/0 through 255.255.255.255/32 + // - ::/0 through ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128 + // In other words, there is no tightly-coupled combination of + // interface address prefixes and connected routes. + stats := new(routeStats) for _, ifa := range ifat { switch ifa := ifa.(type) { case *IPNet: - if ifa == nil || ifa.IP == nil || ifa.IP.IsUnspecified() || ifa.IP.IsMulticast() || ifa.Mask == nil { - t.Errorf("unexpected value: %#v", ifa) - continue + if ifa == nil || ifa.IP == nil || ifa.IP.IsMulticast() || ifa.Mask == nil { + return nil, fmt.Errorf("unexpected value: %#v", ifa) } if len(ifa.IP) != IPv6len { - t.Errorf("should be internal representation either IPv6 or IPv6 IPv4-mapped address: %#v", ifa) - continue + return nil, fmt.Errorf("should be internal representation either IPv6 or IPv4-mapped IPv6 address: %#v", ifa) } prefixLen, maxPrefixLen := ifa.Mask.Size() if ifa.IP.To4() != nil { if 0 >= prefixLen || prefixLen > 8*IPv4len || maxPrefixLen != 8*IPv4len { - t.Errorf("unexpected prefix length: %d/%d", prefixLen, maxPrefixLen) - continue + return nil, fmt.Errorf("unexpected prefix length: %d/%d for %#v", prefixLen, maxPrefixLen, ifa) } if ifa.IP.IsLoopback() && (prefixLen != 8 && prefixLen != 8*IPv4len) { // see RFC 1122 - t.Errorf("unexpected prefix length for IPv4 loopback: %d/%d", prefixLen, maxPrefixLen) - continue + return nil, fmt.Errorf("unexpected prefix length: %d/%d for %#v", prefixLen, maxPrefixLen, ifa) } - naf4++ + stats.ipv4++ } if ifa.IP.To16() != nil && ifa.IP.To4() == nil { if 0 >= prefixLen || prefixLen > 8*IPv6len || maxPrefixLen != 8*IPv6len { - t.Errorf("unexpected prefix length: %d/%d", prefixLen, maxPrefixLen) - continue + return nil, fmt.Errorf("unexpected prefix length: %d/%d for %#v", prefixLen, maxPrefixLen, ifa) } if ifa.IP.IsLoopback() && prefixLen != 8*IPv6len { // see RFC 4291 - t.Errorf("unexpected prefix length for IPv6 loopback: %d/%d", prefixLen, maxPrefixLen) - continue + return nil, fmt.Errorf("unexpected prefix length: %d/%d for %#v", prefixLen, maxPrefixLen, ifa) } - naf6++ + stats.ipv6++ } - t.Logf("interface address %q", ifa.String()) case *IPAddr: - if ifa == nil || ifa.IP == nil || ifa.IP.IsUnspecified() || ifa.IP.IsMulticast() { - t.Errorf("unexpected value: %#v", ifa) - continue + if ifa == nil || ifa.IP == nil || ifa.IP.IsMulticast() { + return nil, fmt.Errorf("unexpected value: %#v", ifa) } if len(ifa.IP) != IPv6len { - t.Errorf("should be internal representation either IPv6 or IPv6 IPv4-mapped address: %#v", ifa) - continue + return nil, fmt.Errorf("should be internal representation either IPv6 or IPv4-mapped IPv6 address: %#v", ifa) } if ifa.IP.To4() != nil { - naf4++ + stats.ipv4++ } if ifa.IP.To16() != nil && ifa.IP.To4() == nil { - naf6++ + stats.ipv6++ } - t.Logf("interface address %s", ifa.String()) default: - t.Errorf("unexpected type: %T", ifa) + return nil, fmt.Errorf("unexpected type: %T", ifa) } } - return + return stats, nil } -func testMulticastAddrs(t *testing.T, ifmat []Addr) (nmaf4, nmaf6 int) { - for _, ifma := range ifmat { - switch ifma := ifma.(type) { +func validateInterfaceMulticastAddrs(ifat []Addr) (*routeStats, error) { + stats := new(routeStats) + for _, ifa := range ifat { + switch ifa := ifa.(type) { case *IPAddr: - if ifma == nil || ifma.IP == nil || ifma.IP.IsUnspecified() || !ifma.IP.IsMulticast() { - t.Errorf("unexpected value: %+v", ifma) - continue + if ifa == nil || ifa.IP == nil || ifa.IP.IsUnspecified() || !ifa.IP.IsMulticast() { + return nil, fmt.Errorf("unexpected value: %#v", ifa) } - if len(ifma.IP) != IPv6len { - t.Errorf("should be internal representation either IPv6 or IPv6 IPv4-mapped address: %#v", ifma) - continue + if len(ifa.IP) != IPv6len { + return nil, fmt.Errorf("should be internal representation either IPv6 or IPv4-mapped IPv6 address: %#v", ifa) } - if ifma.IP.To4() != nil { - nmaf4++ + if ifa.IP.To4() != nil { + stats.ipv4++ } - if ifma.IP.To16() != nil && ifma.IP.To4() == nil { - nmaf6++ + if ifa.IP.To16() != nil && ifa.IP.To4() == nil { + stats.ipv6++ } - t.Logf("joined group address %q", ifma.String()) default: - t.Errorf("unexpected type: %T", ifma) + return nil, fmt.Errorf("unexpected type: %T", ifa) + } + } + return stats, nil +} + +func checkUnicastStats(ifStats *ifStats, uniStats *routeStats) error { + // Test the existence of connected unicast routes for IPv4. + if supportsIPv4 && ifStats.loop+ifStats.other > 0 && uniStats.ipv4 == 0 { + return fmt.Errorf("num IPv4 unicast routes = 0; want >0; summary: %+v, %+v", ifStats, uniStats) + } + // Test the existence of connected unicast routes for IPv6. + // We can assume the existence of ::1/128 when at least one + // loopback interface is installed. + if supportsIPv6 && ifStats.loop > 0 && uniStats.ipv6 == 0 { + return fmt.Errorf("num IPv6 unicast routes = 0; want >0; summary: %+v, %+v", ifStats, uniStats) + } + return nil +} + +func checkMulticastStats(ifStats *ifStats, uniStats, multiStats *routeStats) error { + switch runtime.GOOS { + case "dragonfly", "nacl", "netbsd", "openbsd", "plan9", "solaris": + default: + // Test the existence of connected multicast route + // clones for IPv4. Unlike IPv6, IPv4 multicast + // capability is not a mandatory feature, and so IPv4 + // multicast validation is ignored and we only check + // IPv6 below. + // + // Test the existence of connected multicast route + // clones for IPv6. Some platform never uses loopback + // interface as the nexthop for multicast routing. + // We can assume the existence of connected multicast + // route clones when at least two connected unicast + // routes, ::1/128 and other, are installed. + if supportsIPv6 && ifStats.loop > 0 && uniStats.ipv6 > 1 && multiStats.ipv6 == 0 { + return fmt.Errorf("num IPv6 multicast route clones = 0; want >0; summary: %+v, %+v, %+v", ifStats, uniStats, multiStats) } } - return + return nil } func BenchmarkInterfaces(b *testing.B) { diff --git a/libgo/go/net/interface_windows.go b/libgo/go/net/interface_windows.go index 4d6bcdf..8b976e5 100644 --- a/libgo/go/net/interface_windows.go +++ b/libgo/go/net/interface_windows.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -61,7 +61,7 @@ func adapterAddresses() ([]*windows.IpAdapterAddresses, error) { } // If the ifindex is zero, interfaceTable returns mappings of all -// network interfaces. Otherwise it returns a mapping of a specific +// network interfaces. Otherwise it returns a mapping of a specific // interface. func interfaceTable(ifindex int) ([]Interface, error) { aas, err := adapterAddresses() @@ -116,7 +116,7 @@ func interfaceTable(ifindex int) ([]Interface, error) { } // If the ifi is nil, interfaceAddrTable returns addresses for all -// network interfaces. Otherwise it returns addresses for a specific +// network interfaces. Otherwise it returns addresses for a specific // interface. func interfaceAddrTable(ifi *Interface) ([]Addr, error) { aas, err := adapterAddresses() diff --git a/libgo/go/net/internal/socktest/switch.go b/libgo/go/net/internal/socktest/switch.go index 8bef06b..3c37b6f 100644 --- a/libgo/go/net/internal/socktest/switch.go +++ b/libgo/go/net/internal/socktest/switch.go @@ -121,7 +121,7 @@ const ( FilterSocket FilterType = iota // for Socket FilterConnect // for Connect or ConnectEx FilterListen // for Listen - FilterAccept // for Accept or Accept4 + FilterAccept // for Accept, Accept4 or AcceptEx FilterGetsockoptInt // for GetsockoptInt FilterClose // for Close or Closesocket ) diff --git a/libgo/go/net/internal/socktest/sys_windows.go b/libgo/go/net/internal/socktest/sys_windows.go index e61bf2b..2e3d2bc 100644 --- a/libgo/go/net/internal/socktest/sys_windows.go +++ b/libgo/go/net/internal/socktest/sys_windows.go @@ -154,3 +154,33 @@ func (sw *Switch) Listen(s syscall.Handle, backlog int) (err error) { sw.stats.getLocked(so.Cookie).Listened++ return nil } + +// AcceptEx wraps syscall.AcceptEx. +func (sw *Switch) AcceptEx(ls syscall.Handle, as syscall.Handle, b *byte, rxdatalen uint32, laddrlen uint32, raddrlen uint32, rcvd *uint32, overlapped *syscall.Overlapped) error { + so := sw.sockso(ls) + if so == nil { + return syscall.AcceptEx(ls, as, b, rxdatalen, laddrlen, raddrlen, rcvd, overlapped) + } + sw.fmu.RLock() + f, _ := sw.fltab[FilterAccept] + sw.fmu.RUnlock() + + af, err := f.apply(so) + if err != nil { + return err + } + so.Err = syscall.AcceptEx(ls, as, b, rxdatalen, laddrlen, raddrlen, rcvd, overlapped) + if err = af.apply(so); err != nil { + return err + } + + sw.smu.Lock() + defer sw.smu.Unlock() + if so.Err != nil { + sw.stats.getLocked(so.Cookie).AcceptFailed++ + return so.Err + } + nso := sw.addLocked(as, so.Cookie.Family(), so.Cookie.Type(), so.Cookie.Protocol()) + sw.stats.getLocked(nso.Cookie).Accepted++ + return nil +} diff --git a/libgo/go/net/ip.go b/libgo/go/net/ip.go index cc004d6..d0c8263 100644 --- a/libgo/go/net/ip.go +++ b/libgo/go/net/ip.go @@ -252,9 +252,11 @@ func (ip IP) Mask(mask IPMask) IP { } // String returns the string form of the IP address ip. -// If the address is an IPv4 address, the string representation -// is dotted decimal ("74.125.19.99"). Otherwise the representation -// is IPv6 ("2001:4860:0:2001::68"). +// It returns one of 4 forms: +// - "<nil>", if ip has length 0 +// - dotted decimal ("192.0.2.1"), if ip is an IPv4 or IP4-mapped IPv6 address +// - IPv6 ("2001:db8::1"), if ip is a valid IPv6 address +// - the hexadecimal form of ip, without punctuation, if no other cases apply func (ip IP) String() string { p := ip @@ -270,7 +272,7 @@ func (ip IP) String() string { uitoa(uint(p4[3])) } if len(p) != IPv6len { - return "?" + return "?" + hexString(ip) } // Find longest run of zeros. @@ -312,6 +314,14 @@ func (ip IP) String() string { return string(b) } +func hexString(b []byte) string { + s := make([]byte, len(b)*2) + for i, tn := range b { + s[i*2], s[i*2+1] = hexDigit[tn>>4], hexDigit[tn&0xf] + } + return string(s) +} + // ipEmptyString is like ip.String except that it returns // an empty string when ip is unset. func ipEmptyString(ip IP) string { @@ -328,7 +338,7 @@ func (ip IP) MarshalText() ([]byte, error) { return []byte(""), nil } if len(ip) != IPv4len && len(ip) != IPv6len { - return nil, &AddrError{Err: "invalid IP address", Addr: ip.String()} + return nil, &AddrError{Err: "invalid IP address", Addr: hexString(ip)} } return []byte(ip.String()), nil } @@ -377,6 +387,10 @@ func bytesEqual(x, y []byte) bool { return true } +func (ip IP) matchAddrFamily(x IP) bool { + return ip.To4() != nil && x.To4() != nil || ip.To16() != nil && ip.To4() == nil && x.To16() != nil && x.To4() == nil +} + // If mask is a sequence of 1 bits followed by 0 bits, // return the number of 1 bits. func simpleMaskLength(mask IPMask) int { @@ -422,11 +436,7 @@ func (m IPMask) String() string { if len(m) == 0 { return "<nil>" } - buf := make([]byte, len(m)*2) - for i, b := range m { - buf[i*2], buf[i*2+1] = hexDigit[b>>4], hexDigit[b&0xf] - } - return string(buf) + return hexString(m) } func networkNumberAndMask(n *IPNet) (ip IP, m IPMask) { @@ -473,12 +483,12 @@ func (n *IPNet) Contains(ip IP) bool { // Network returns the address's network name, "ip+net". func (n *IPNet) Network() string { return "ip+net" } -// String returns the CIDR notation of n like "192.168.100.1/24" -// or "2001:DB8::/48" as defined in RFC 4632 and RFC 4291. +// String returns the CIDR notation of n like "192.0.2.1/24" +// or "2001:db8::/48" as defined in RFC 4632 and RFC 4291. // If the mask is not in the canonical form, it returns the // string which consists of an IP address, followed by a slash // character and a mask expressed as hexadecimal form with no -// punctuation like "192.168.100.1/c000ff00". +// punctuation like "198.51.100.1/c000ff00". func (n *IPNet) String() string { nn, m := networkNumberAndMask(n) if nn == nil || m == nil { @@ -631,8 +641,8 @@ func parseIPv6(s string, zoneAllowed bool) (ip IP, zone string) { } // ParseIP parses s as an IP address, returning the result. -// The string s can be in dotted decimal ("74.125.19.99") -// or IPv6 ("2001:4860:0:2001::68") form. +// The string s can be in dotted decimal ("192.0.2.1") +// or IPv6 ("2001:db8::68") form. // If s is not a valid textual representation of an IP address, // ParseIP returns nil. func ParseIP(s string) IP { @@ -649,12 +659,12 @@ func ParseIP(s string) IP { } // ParseCIDR parses s as a CIDR notation IP address and mask, -// like "192.168.100.1/24" or "2001:DB8::/48", as defined in +// like "192.0.2.0/24" or "2001:db8::/32", as defined in // RFC 4632 and RFC 4291. // // It returns the IP address and the network implied by the IP -// and mask. For example, ParseCIDR("192.168.100.1/16") returns -// the IP address 192.168.100.1 and the network 192.168.0.0/16. +// and mask. For example, ParseCIDR("198.51.100.1/24") returns +// the IP address 198.51.100.1 and the network 198.51.100.0/24. func ParseCIDR(s string) (IP, *IPNet, error) { i := byteIndex(s, '/') if i < 0 { diff --git a/libgo/go/net/ip_test.go b/libgo/go/net/ip_test.go index 3d95a73..b6ac26d 100644 --- a/libgo/go/net/ip_test.go +++ b/libgo/go/net/ip_test.go @@ -5,6 +5,7 @@ package net import ( + "bytes" "reflect" "runtime" "testing" @@ -124,30 +125,119 @@ func TestMarshalEmptyIP(t *testing.T) { } var ipStringTests = []struct { - in IP - out string // see RFC 5952 + in IP // see RFC 791 and RFC 4291 + str string // see RFC 791, RFC 4291 and RFC 5952 + byt []byte + error }{ - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, "2001:db8::123:12:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1}, "2001:db8:0:1:0:1:0:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0}, "2001:db8:1:0:1:0:1:0"}, - {IP{0x20, 0x1, 0, 0, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001::1:0:0:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0}, "2001:db8:0:0:1::"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1:0:0:1"}, - {IP{0x20, 0x1, 0xD, 0xB8, 0, 0, 0, 0, 0, 0xA, 0, 0xB, 0, 0xC, 0, 0xD}, "2001:db8::a:b:c:d"}, - {IPv4(192, 168, 0, 1), "192.168.0.1"}, - {nil, ""}, + // IPv4 address + { + IP{192, 0, 2, 1}, + "192.0.2.1", + []byte("192.0.2.1"), + nil, + }, + { + IP{0, 0, 0, 0}, + "0.0.0.0", + []byte("0.0.0.0"), + nil, + }, + + // IPv4-mapped IPv6 address + { + IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 192, 0, 2, 1}, + "192.0.2.1", + []byte("192.0.2.1"), + nil, + }, + { + IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0, 0, 0}, + "0.0.0.0", + []byte("0.0.0.0"), + nil, + }, + + // IPv6 address + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, + "2001:db8::123:12:1", + []byte("2001:db8::123:12:1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x1}, + "2001:db8::1", + []byte("2001:db8::1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1}, + "2001:db8:0:1:0:1:0:1", + []byte("2001:db8:0:1:0:1:0:1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0}, + "2001:db8:1:0:1:0:1:0", + []byte("2001:db8:1:0:1:0:1:0"), + nil, + }, + { + IP{0x20, 0x1, 0, 0, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, + "2001::1:0:0:1", + []byte("2001::1:0:0:1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0}, + "2001:db8:0:0:1::", + []byte("2001:db8:0:0:1::"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, + "2001:db8::1:0:0:1", + []byte("2001:db8::1:0:0:1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0xa, 0, 0xb, 0, 0xc, 0, 0xd}, + "2001:db8::a:b:c:d", + []byte("2001:db8::a:b:c:d"), + nil, + }, + { + IPv6unspecified, + "::", + []byte("::"), + nil, + }, + + // IP wildcard equivalent address in Dial/Listen API + { + nil, + "<nil>", + nil, + nil, + }, + + // Opaque byte sequence + { + IP{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + "?0123456789abcdef", + nil, + &AddrError{Err: "invalid IP address", Addr: "0123456789abcdef"}, + }, } func TestIPString(t *testing.T) { for _, tt := range ipStringTests { - if tt.in != nil { - if out := tt.in.String(); out != tt.out { - t.Errorf("IP.String(%v) = %q, want %q", tt.in, out, tt.out) - } + if out := tt.in.String(); out != tt.str { + t.Errorf("IP.String(%v) = %q, want %q", tt.in, out, tt.str) } - if out, err := tt.in.MarshalText(); string(out) != tt.out || err != nil { - t.Errorf("IP.MarshalText(%v) = %q, %v, want %q, nil", tt.in, out, err, tt.out) + if out, err := tt.in.MarshalText(); !bytes.Equal(out, tt.byt) || !reflect.DeepEqual(err, tt.error) { + t.Errorf("IP.MarshalText(%v) = %v, %v, want %v, %v", tt.in, out, err, tt.byt, tt.error) } } } @@ -379,8 +469,8 @@ var splitJoinTests = []struct { {"", "0", ":0"}, {"google.com", "https%foo", "google.com:https%foo"}, // Go 1.0 behavior - {"127.0.0.1", "", "127.0.0.1:"}, // Go 1.0 behaviour - {"www.google.com", "", "www.google.com:"}, // Go 1.0 behaviour + {"127.0.0.1", "", "127.0.0.1:"}, // Go 1.0 behavior + {"www.google.com", "", "www.google.com:"}, // Go 1.0 behavior } var splitFailureTests = []struct { @@ -421,13 +511,16 @@ func TestSplitHostPort(t *testing.T) { } } for _, tt := range splitFailureTests { - if _, _, err := SplitHostPort(tt.hostPort); err == nil { + if host, port, err := SplitHostPort(tt.hostPort); err == nil { t.Errorf("SplitHostPort(%q) should have failed", tt.hostPort) } else { e := err.(*AddrError) if e.Err != tt.err { t.Errorf("SplitHostPort(%q) = _, _, %q; want %q", tt.hostPort, e.Err, tt.err) } + if host != "" || port != "" { + t.Errorf("SplitHostPort(%q) = %q, %q, err; want %q, %q, err on failure", tt.hostPort, host, port, "", "") + } } } } diff --git a/libgo/go/net/iprawsock.go b/libgo/go/net/iprawsock.go index f02df7f..173b3cb 100644 --- a/libgo/go/net/iprawsock.go +++ b/libgo/go/net/iprawsock.go @@ -1,9 +1,14 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import ( + "context" + "syscall" +) + // IPAddr represents the address of an IP end point. type IPAddr struct { IP IP @@ -45,7 +50,7 @@ func ResolveIPAddr(net, addr string) (*IPAddr, error) { if net == "" { // a hint wildcard for Go 1.0 undocumented behavior net = "ip" } - afnet, _, err := parseNetwork(net) + afnet, _, err := parseNetwork(context.Background(), net) if err != nil { return nil, err } @@ -54,9 +59,136 @@ func ResolveIPAddr(net, addr string) (*IPAddr, error) { default: return nil, UnknownNetworkError(net) } - addrs, err := internetAddrList(afnet, addr, noDeadline) + addrs, err := internetAddrList(context.Background(), afnet, addr) if err != nil { return nil, err } return addrs.first(isIPv4).(*IPAddr), nil } + +// IPConn is the implementation of the Conn and PacketConn interfaces +// for IP network connections. +type IPConn struct { + conn +} + +// ReadFromIP reads an IP packet from c, copying the payload into b. +// It returns the number of bytes copied into b and the return address +// that was on the packet. +// +// ReadFromIP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, addr, err +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + if addr == nil { + return n, nil, err + } + return n, addr, err +} + +// ReadMsgIP reads a packet from c, copying the payload into b and the +// associated out-of-band data into oob. It returns the number of +// bytes copied into b, the number of bytes copied into oob, the flags +// that were set on the packet and the source address of the packet. +func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { + if !c.ok() { + return 0, 0, 0, nil, syscall.EINVAL + } + n, oobn, flags, addr, err = c.readMsg(b, oob) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return +} + +// WriteToIP writes an IP packet to addr via c, copying the payload +// from b. +// +// WriteToIP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.writeTo(b, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return n, err +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*IPAddr) + if !ok { + return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} + } + n, err := c.writeTo(b, a) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: a.opAddr(), Err: err} + } + return n, err +} + +// WriteMsgIP writes a packet to addr via c, copying the payload from +// b and the associated out-of-band data from oob. It returns the +// number of payload and out-of-band bytes written. +func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { + if !c.ok() { + return 0, 0, syscall.EINVAL + } + n, oobn, err = c.writeMsg(b, oob, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return +} + +func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} } + +// DialIP connects to the remote address raddr on the network protocol +// netProto, which must be "ip", "ip4", or "ip6" followed by a colon +// and a protocol number or name. +func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) { + c, err := dialIP(context.Background(), netProto, laddr, raddr) + if err != nil { + return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + } + return c, nil +} + +// ListenIP listens for incoming IP packets addressed to the local +// address laddr. The returned connection's ReadFrom and WriteTo +// methods can be used to receive and send IP packets with per-packet +// addressing. +func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) { + c, err := listenIP(context.Background(), netProto, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return c, nil +} diff --git a/libgo/go/net/iprawsock_plan9.go b/libgo/go/net/iprawsock_plan9.go index b027adc..6aebea1 100644 --- a/libgo/go/net/iprawsock_plan9.go +++ b/libgo/go/net/iprawsock_plan9.go @@ -1,82 +1,34 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "syscall" - "time" ) -// IPConn is the implementation of the Conn and PacketConn interfaces -// for IP network connections. -type IPConn struct { - conn +func (c *IPConn) readFrom(b []byte) (int, *IPAddr, error) { + return 0, nil, syscall.EPLAN9 } -// ReadFromIP reads an IP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *IPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { + return 0, 0, 0, nil, syscall.EPLAN9 } -// ReadFrom implements the PacketConn ReadFrom method. -func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *IPConn) writeTo(b []byte, addr *IPAddr) (int, error) { + return 0, syscall.EPLAN9 } -// ReadMsgIP reads a packet from c, copying the payload into b and the -// associated out-of-band data into oob. It returns the number of -// bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet and the source address of the packet. -func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { - return 0, 0, 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *IPConn) writeMsg(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { + return 0, 0, syscall.EPLAN9 } -// WriteToIP writes an IP packet to addr via c, copying the payload -// from b. -// -// WriteToIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} +func dialIP(ctx context.Context, netProto string, laddr, raddr *IPAddr) (*IPConn, error) { + return nil, syscall.EPLAN9 } -// WriteTo implements the PacketConn WriteTo method. -func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr, Err: syscall.EPLAN9} -} - -// WriteMsgIP writes a packet to addr via c, copying the payload from -// b and the associated out-of-band data from oob. It returns the -// number of payload and out-of-band bytes written. -func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { - return 0, 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} -} - -// DialIP connects to the remote address raddr on the network protocol -// netProto, which must be "ip", "ip4", or "ip6" followed by a colon -// and a protocol number or name. -func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) { - return dialIP(netProto, laddr, raddr, noDeadline) -} - -func dialIP(netProto string, laddr, raddr *IPAddr, deadline time.Time) (*IPConn, error) { - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: syscall.EPLAN9} -} - -// ListenIP listens for incoming IP packets addressed to the local -// address laddr. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send IP packets with per-packet -// addressing. -func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) { - return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: syscall.EPLAN9} +func listenIP(ctx context.Context, netProto string, laddr *IPAddr) (*IPConn, error) { + return nil, syscall.EPLAN9 } diff --git a/libgo/go/net/iprawsock_posix.go b/libgo/go/net/iprawsock_posix.go index 93fee3e..3e0b060 100644 --- a/libgo/go/net/iprawsock_posix.go +++ b/libgo/go/net/iprawsock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,8 +7,8 @@ package net import ( + "context" "syscall" - "time" ) // BUG(mikio): On every POSIX platform, reads from the "ip4" network @@ -50,25 +50,7 @@ func (a *IPAddr) sockaddr(family int) (syscall.Sockaddr, error) { return ipToSockaddr(family, a.IP, 0, a.Zone) } -// IPConn is the implementation of the Conn and PacketConn interfaces -// for IP network connections. -type IPConn struct { - conn -} - -func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} } - -// ReadFromIP reads an IP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } +func (c *IPConn) readFrom(b []byte) (int, *IPAddr, error) { // TODO(cw,rsc): consider using readv if we know the family // type to avoid the header trim/copy var addr *IPAddr @@ -80,9 +62,6 @@ func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { case *syscall.SockaddrInet6: addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return n, addr, err } @@ -101,26 +80,7 @@ func stripIPv4Header(n int, b []byte) int { return n - l } -// ReadFrom implements the PacketConn ReadFrom method. -func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } - n, addr, err := c.ReadFromIP(b) - if addr == nil { - return n, nil, err - } - return n, addr, err -} - -// ReadMsgIP reads a packet from c, copying the payload into b and the -// associated out-of-band data into oob. It returns the number of -// bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet and the source address of the packet. -func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { - if !c.ok() { - return 0, 0, 0, nil, syscall.EINVAL - } +func (c *IPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { var sa syscall.Sockaddr n, oobn, flags, sa, err = c.fd.readMsg(b, oob) switch sa := sa.(type) { @@ -129,121 +89,70 @@ func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err case *syscall.SockaddrInet6: addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return } -// WriteToIP writes an IP packet to addr via c, copying the payload -// from b. -// -// WriteToIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } +func (c *IPConn) writeTo(b []byte, addr *IPAddr) (int, error) { if c.fd.isConnected { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, ErrWriteToConnected } if addr == nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} + return 0, errMissingAddress } sa, err := addr.sockaddr(c.fd.family) if err != nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - n, err := c.fd.writeTo(b, sa) - if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - return n, err -} - -// WriteTo implements the PacketConn WriteTo method. -func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } - a, ok := addr.(*IPAddr) - if !ok { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} + return 0, err } - return c.WriteToIP(b, a) + return c.fd.writeTo(b, sa) } -// WriteMsgIP writes a packet to addr via c, copying the payload from -// b and the associated out-of-band data from oob. It returns the -// number of payload and out-of-band bytes written. -func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { - if !c.ok() { - return 0, 0, syscall.EINVAL - } +func (c *IPConn) writeMsg(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { if c.fd.isConnected { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, 0, ErrWriteToConnected } if addr == nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} - } - var sa syscall.Sockaddr - sa, err = addr.sockaddr(c.fd.family) - if err != nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + return 0, 0, errMissingAddress } - n, oobn, err = c.fd.writeMsg(b, oob, sa) + sa, err := addr.sockaddr(c.fd.family) if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + return 0, 0, err } - return -} - -// DialIP connects to the remote address raddr on the network protocol -// netProto, which must be "ip", "ip4", or "ip6" followed by a colon -// and a protocol number or name. -func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) { - return dialIP(netProto, laddr, raddr, noDeadline) + return c.fd.writeMsg(b, oob, sa) } -func dialIP(netProto string, laddr, raddr *IPAddr, deadline time.Time) (*IPConn, error) { - net, proto, err := parseNetwork(netProto) +func dialIP(ctx context.Context, netProto string, laddr, raddr *IPAddr) (*IPConn, error) { + network, proto, err := parseNetwork(ctx, netProto) if err != nil { - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, err } - switch net { + switch network { case "ip", "ip4", "ip6": default: - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(netProto)} + return nil, UnknownNetworkError(netProto) } if raddr == nil { - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + return nil, errMissingAddress } - fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_RAW, proto, "dial", noCancel) + fd, err := internetSocket(ctx, network, laddr, raddr, syscall.SOCK_RAW, proto, "dial") if err != nil { - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, err } return newIPConn(fd), nil } -// ListenIP listens for incoming IP packets addressed to the local -// address laddr. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send IP packets with per-packet -// addressing. -func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) { - net, proto, err := parseNetwork(netProto) +func listenIP(ctx context.Context, netProto string, laddr *IPAddr) (*IPConn, error) { + network, proto, err := parseNetwork(ctx, netProto) if err != nil { - return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, err } - switch net { + switch network { case "ip", "ip4", "ip6": default: - return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(netProto)} + return nil, UnknownNetworkError(netProto) } - fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_RAW, proto, "listen", noCancel) + fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_RAW, proto, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, err } return newIPConn(fd), nil } diff --git a/libgo/go/net/ipraw_test.go b/libgo/go/net/iprawsock_test.go index 5d86a9d..29cd4b6 100644 --- a/libgo/go/net/ipraw_test.go +++ b/libgo/go/net/iprawsock_test.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/ipsock.go b/libgo/go/net/ipsock.go index 55f697f..24daf17 100644 --- a/libgo/go/net/ipsock.go +++ b/libgo/go/net/ipsock.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,8 +7,7 @@ package net import ( - "errors" - "time" + "context" ) var ( @@ -22,7 +21,7 @@ var ( // supportsIPv4map reports whether the platform supports // mapping an IPv4 address inside an IPv6 address at transport - // layer protocols. See RFC 4291, RFC 4038 and RFC 3493. + // layer protocols. See RFC 4291, RFC 4038 and RFC 3493. supportsIPv4map bool ) @@ -73,8 +72,6 @@ func (addrs addrList) partition(strategy func(Addr) bool) (primaries, fallbacks return } -var errNoSuitableAddress = errors.New("no suitable address found") - // filterAddrList applies a filter to a list of IP addresses, // yielding a list of Addr objects. Known filters are nil, ipv4only, // and ipv6only. It returns every address when the filter is nil. @@ -106,73 +103,65 @@ func ipv6only(addr IPAddr) bool { // SplitHostPort splits a network address of the form "host:port", // "[host]:port" or "[ipv6-host%zone]:port" into host or -// ipv6-host%zone and port. A literal address or host name for IPv6 +// ipv6-host%zone and port. A literal address or host name for IPv6 // must be enclosed in square brackets, as in "[::1]:80", // "[ipv6-host]:http" or "[ipv6-host%zone]:80". func SplitHostPort(hostport string) (host, port string, err error) { + const ( + missingPort = "missing port in address" + tooManyColons = "too many colons in address" + ) + addrErr := func(addr, why string) (host, port string, err error) { + return "", "", &AddrError{Err: why, Addr: addr} + } j, k := 0, 0 // The port starts after the last colon. i := last(hostport, ':') if i < 0 { - goto missingPort + return addrErr(hostport, missingPort) } if hostport[0] == '[' { // Expect the first ']' just before the last ':'. end := byteIndex(hostport, ']') if end < 0 { - err = &AddrError{Err: "missing ']' in address", Addr: hostport} - return + return addrErr(hostport, "missing ']' in address") } switch end + 1 { case len(hostport): // There can't be a ':' behind the ']' now. - goto missingPort + return addrErr(hostport, missingPort) case i: // The expected result. default: // Either ']' isn't followed by a colon, or it is // followed by a colon that is not the last one. if hostport[end+1] == ':' { - goto tooManyColons + return addrErr(hostport, tooManyColons) } - goto missingPort + return addrErr(hostport, missingPort) } host = hostport[1:end] j, k = 1, end+1 // there can't be a '[' resp. ']' before these positions } else { host = hostport[:i] if byteIndex(host, ':') >= 0 { - goto tooManyColons + return addrErr(hostport, tooManyColons) } if byteIndex(host, '%') >= 0 { - goto missingBrackets + return addrErr(hostport, "missing brackets in address") } } if byteIndex(hostport[j:], '[') >= 0 { - err = &AddrError{Err: "unexpected '[' in address", Addr: hostport} - return + return addrErr(hostport, "unexpected '[' in address") } if byteIndex(hostport[k:], ']') >= 0 { - err = &AddrError{Err: "unexpected ']' in address", Addr: hostport} - return + return addrErr(hostport, "unexpected ']' in address") } port = hostport[i+1:] - return - -missingPort: - err = &AddrError{Err: "missing port in address", Addr: hostport} - return - -tooManyColons: - err = &AddrError{Err: "too many colons in address", Addr: hostport} - return - -missingBrackets: - err = &AddrError{Err: "missing brackets in address", Addr: hostport} - return + return host, port, nil } func splitHostZone(s string) (host, zone string) { @@ -201,7 +190,7 @@ func JoinHostPort(host, port string) string { // address or a DNS name, and returns a list of internet protocol // family addresses. The result contains at least one address when // error is nil. -func internetAddrList(net, addr string, deadline time.Time) (addrList, error) { +func internetAddrList(ctx context.Context, net, addr string) (addrList, error) { var ( err error host, port string @@ -249,7 +238,7 @@ func internetAddrList(net, addr string, deadline time.Time) (addrList, error) { return addrList{inetaddr(IPAddr{IP: ip, Zone: zone})}, nil } // Try as a DNS name. - ips, err := lookupIPDeadline(host, deadline) + ips, err := lookupIPContext(ctx, host) if err != nil { return nil, err } @@ -262,24 +251,3 @@ func internetAddrList(net, addr string, deadline time.Time) (addrList, error) { } return filterAddrList(filter, ips, inetaddr) } - -func zoneToString(zone int) string { - if zone == 0 { - return "" - } - if ifi, err := InterfaceByIndex(zone); err == nil { - return ifi.Name - } - return uitoa(uint(zone)) -} - -func zoneToInt(zone string) int { - if zone == "" { - return 0 - } - if ifi, err := InterfaceByName(zone); err == nil { - return ifi.Index - } - n, _, _ := dtoi(zone, 0) - return n -} diff --git a/libgo/go/net/ipsock_plan9.go b/libgo/go/net/ipsock_plan9.go index 9da6ec3..2b84683 100644 --- a/libgo/go/net/ipsock_plan9.go +++ b/libgo/go/net/ipsock_plan9.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,6 +7,7 @@ package net import ( + "context" "os" "syscall" ) @@ -39,8 +40,8 @@ func probeIPv4Stack() bool { return probe(netdir+"/iproute", "4i") } -// probeIPv6Stack returns two boolean values. If the first boolean -// value is true, kernel supports basic IPv6 functionality. If the +// probeIPv6Stack returns two boolean values. If the first boolean +// value is true, kernel supports basic IPv6 functionality. If the // second boolean value is true, kernel supports IPv6 IPv4-mapping. func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { // Plan 9 uses IPv6 natively, see ip(3). @@ -99,7 +100,7 @@ func readPlan9Addr(proto, filename string) (addr Addr, err error) { return addr, nil } -func startPlan9(net string, addr Addr) (ctl *os.File, dest, proto, name string, err error) { +func startPlan9(ctx context.Context, net string, addr Addr) (ctl *os.File, dest, proto, name string, err error) { var ( ip IP port int @@ -118,7 +119,7 @@ func startPlan9(net string, addr Addr) (ctl *os.File, dest, proto, name string, return } - clone, dest, err := queryCS1(proto, ip, port) + clone, dest, err := queryCS1(ctx, proto, ip, port) if err != nil { return } @@ -135,8 +136,8 @@ func startPlan9(net string, addr Addr) (ctl *os.File, dest, proto, name string, return f, dest, proto, string(buf[:n]), nil } -func netErr(e error) { - oe, ok := e.(*OpError) +func fixErr(err error) { + oe, ok := err.(*OpError) if !ok { return } @@ -165,46 +166,71 @@ func netErr(e error) { } } -func dialPlan9(net string, laddr, raddr Addr) (fd *netFD, err error) { - defer func() { netErr(err) }() - f, dest, proto, name, err := startPlan9(net, raddr) +func dialPlan9(ctx context.Context, net string, laddr, raddr Addr) (fd *netFD, err error) { + defer func() { fixErr(err) }() + type res struct { + fd *netFD + err error + } + resc := make(chan res) + go func() { + testHookDialChannel() + fd, err := dialPlan9Blocking(ctx, net, laddr, raddr) + select { + case resc <- res{fd, err}: + case <-ctx.Done(): + if fd != nil { + fd.Close() + } + } + }() + select { + case res := <-resc: + return res.fd, res.err + case <-ctx.Done(): + return nil, mapErr(ctx.Err()) + } +} + +func dialPlan9Blocking(ctx context.Context, net string, laddr, raddr Addr) (fd *netFD, err error) { + f, dest, proto, name, err := startPlan9(ctx, net, raddr) if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr, Addr: raddr, Err: err} + return nil, err } _, err = f.WriteString("connect " + dest) if err != nil { f.Close() - return nil, &OpError{Op: "dial", Net: f.Name(), Source: laddr, Addr: raddr, Err: err} + return nil, err } data, err := os.OpenFile(netdir+"/"+proto+"/"+name+"/data", os.O_RDWR, 0) if err != nil { f.Close() - return nil, &OpError{Op: "dial", Net: net, Source: laddr, Addr: raddr, Err: err} + return nil, err } laddr, err = readPlan9Addr(proto, netdir+"/"+proto+"/"+name+"/local") if err != nil { data.Close() f.Close() - return nil, &OpError{Op: "dial", Net: proto, Source: laddr, Addr: raddr, Err: err} + return nil, err } return newFD(proto, name, f, data, laddr, raddr) } -func listenPlan9(net string, laddr Addr) (fd *netFD, err error) { - defer func() { netErr(err) }() - f, dest, proto, name, err := startPlan9(net, laddr) +func listenPlan9(ctx context.Context, net string, laddr Addr) (fd *netFD, err error) { + defer func() { fixErr(err) }() + f, dest, proto, name, err := startPlan9(ctx, net, laddr) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } _, err = f.WriteString("announce " + dest) if err != nil { f.Close() - return nil, &OpError{Op: "announce", Net: proto, Source: nil, Addr: laddr, Err: err} + return nil, err } laddr, err = readPlan9Addr(proto, netdir+"/"+proto+"/"+name+"/local") if err != nil { f.Close() - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } return newFD(proto, name, f, nil, laddr, nil) } @@ -214,32 +240,32 @@ func (fd *netFD) netFD() (*netFD, error) { } func (fd *netFD) acceptPlan9() (nfd *netFD, err error) { - defer func() { netErr(err) }() + defer func() { fixErr(err) }() if err := fd.readLock(); err != nil { return nil, err } defer fd.readUnlock() f, err := os.Open(fd.dir + "/listen") if err != nil { - return nil, &OpError{Op: "accept", Net: fd.dir + "/listen", Source: nil, Addr: fd.laddr, Err: err} + return nil, err } var buf [16]byte n, err := f.Read(buf[:]) if err != nil { f.Close() - return nil, &OpError{Op: "accept", Net: fd.dir + "/listen", Source: nil, Addr: fd.laddr, Err: err} + return nil, err } name := string(buf[:n]) data, err := os.OpenFile(netdir+"/"+fd.net+"/"+name+"/data", os.O_RDWR, 0) if err != nil { f.Close() - return nil, &OpError{Op: "accept", Net: fd.net, Source: nil, Addr: fd.laddr, Err: err} + return nil, err } raddr, err := readPlan9Addr(fd.net, netdir+"/"+fd.net+"/"+name+"/remote") if err != nil { data.Close() f.Close() - return nil, &OpError{Op: "accept", Net: fd.net, Source: nil, Addr: fd.laddr, Err: err} + return nil, err } return newFD(fd.net, name, f, data, fd.laddr, raddr) } diff --git a/libgo/go/net/ipsock_posix.go b/libgo/go/net/ipsock_posix.go index 2bddd46..abe90ac 100644 --- a/libgo/go/net/ipsock_posix.go +++ b/libgo/go/net/ipsock_posix.go @@ -1,17 +1,15 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows -// Internet protocol family sockets for POSIX - package net import ( + "context" "runtime" "syscall" - "time" ) // BUG(rsc,mikio): On DragonFly BSD and OpenBSD, listening on the @@ -35,15 +33,15 @@ func probeIPv4Stack() bool { // Should we try to use the IPv4 socket interface if we're // only dealing with IPv4 sockets? As long as the host system // understands IPv6, it's okay to pass IPv4 addresses to the IPv6 -// interface. That simplifies our code and is most general. +// interface. That simplifies our code and is most general. // Unfortunately, we need to run on kernels built without IPv6 -// support too. So probe the kernel to figure it out. +// support too. So probe the kernel to figure it out. // // probeIPv6Stack probes both basic IPv6 capability and IPv6 IPv4- // mapping capability which is controlled by IPV6_V6ONLY socket // option and/or kernel state "net.inet6.ip6.v6only". -// It returns two boolean values. If the first boolean value is -// true, kernel supports basic IPv6 functionality. If the second +// It returns two boolean values. If the first boolean value is +// true, kernel supports basic IPv6 functionality. If the second // boolean value is true, kernel supports IPv6 IPv4-mapping. func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { var probes = []struct { @@ -52,7 +50,7 @@ func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { }{ // IPv6 communication capability {laddr: TCPAddr{IP: ParseIP("::1")}, value: 1}, - // IPv6 IPv4-mapped address communication capability + // IPv4-mapped IPv6 address communication capability {laddr: TCPAddr{IP: IPv4(127, 0, 0, 1)}, value: 0}, } var supps [2]bool @@ -61,7 +59,7 @@ func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { // Some released versions of DragonFly BSD pretend to // accept IPV6_V6ONLY=0 successfully, but the state // still stays IPV6_V6ONLY=1. Eventually DragonFly BSD - // stops preteding, but the transition period would + // stops pretending, but the transition period would // cause unpredictable behavior and we need to avoid // it. // @@ -93,8 +91,8 @@ func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { } // favoriteAddrFamily returns the appropriate address family to -// the given net, laddr, raddr and mode. At first it figures -// address family out from the net. If mode indicates "listen" +// the given net, laddr, raddr and mode. At first it figures +// address family out from the net. If mode indicates "listen" // and laddr is a wildcard, it assumes that the user wants to // make a passive connection with a wildcard address family, both // AF_INET and AF_INET6, and a wildcard address like following: @@ -155,10 +153,9 @@ func favoriteAddrFamily(net string, laddr, raddr sockaddr, mode string) (family } // Internet sockets (TCP, UDP, IP) - -func internetSocket(net string, laddr, raddr sockaddr, deadline time.Time, sotype, proto int, mode string, cancel <-chan struct{}) (fd *netFD, err error) { +func internetSocket(ctx context.Context, net string, laddr, raddr sockaddr, sotype, proto int, mode string) (fd *netFD, err error) { family, ipv6only := favoriteAddrFamily(net, laddr, raddr, mode) - return socket(net, family, sotype, proto, ipv6only, laddr, raddr, deadline, cancel) + return socket(ctx, net, family, sotype, proto, ipv6only, laddr, raddr) } func ipToSockaddr(family int, ip IP, port int, zone string) (syscall.Sockaddr, error) { @@ -167,34 +164,35 @@ func ipToSockaddr(family int, ip IP, port int, zone string) (syscall.Sockaddr, e if len(ip) == 0 { ip = IPv4zero } - if ip = ip.To4(); ip == nil { + ip4 := ip.To4() + if ip4 == nil { return nil, &AddrError{Err: "non-IPv4 address", Addr: ip.String()} } - sa := new(syscall.SockaddrInet4) - for i := 0; i < IPv4len; i++ { - sa.Addr[i] = ip[i] - } - sa.Port = port + sa := &syscall.SockaddrInet4{Port: port} + copy(sa.Addr[:], ip4) return sa, nil case syscall.AF_INET6: - if len(ip) == 0 { - ip = IPv6zero - } - // IPv4 callers use 0.0.0.0 to mean "announce on any available address". - // In IPv6 mode, Linux treats that as meaning "announce on 0.0.0.0", - // which it refuses to do. Rewrite to the IPv6 unspecified address. - if ip.Equal(IPv4zero) { + // In general, an IP wildcard address, which is either + // "0.0.0.0" or "::", means the entire IP addressing + // space. For some historical reason, it is used to + // specify "any available address" on some operations + // of IP node. + // + // When the IP node supports IPv4-mapped IPv6 address, + // we allow an listener to listen to the wildcard + // address of both IP addressing spaces by specifying + // IPv6 wildcard address. + if len(ip) == 0 || ip.Equal(IPv4zero) { ip = IPv6zero } - if ip = ip.To16(); ip == nil { + // We accept any IPv6 address including IPv4-mapped + // IPv6 address. + ip6 := ip.To16() + if ip6 == nil { return nil, &AddrError{Err: "non-IPv6 address", Addr: ip.String()} } - sa := new(syscall.SockaddrInet6) - for i := 0; i < IPv6len; i++ { - sa.Addr[i] = ip[i] - } - sa.Port = port - sa.ZoneId = uint32(zoneToInt(zone)) + sa := &syscall.SockaddrInet6{Port: port, ZoneId: uint32(zoneToInt(zone))} + copy(sa.Addr[:], ip6) return sa, nil } return nil, &AddrError{Err: "invalid address family", Addr: ip.String()} diff --git a/libgo/go/net/listen_test.go b/libgo/go/net/listen_test.go index 51ffe67..6037f36 100644 --- a/libgo/go/net/listen_test.go +++ b/libgo/go/net/listen_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -8,6 +8,7 @@ package net import ( "fmt" + "internal/testenv" "os" "runtime" "syscall" @@ -157,7 +158,7 @@ var dualStackTCPListenerTests = []struct { network2, address2 string // second listener xerr error // expected error value, nil or other }{ - // Test cases and expected results for the attemping 2nd listen on the same port + // Test cases and expected results for the attempting 2nd listen on the same port // 1st listen 2nd listen darwin freebsd linux openbsd // ------------------------------------------------------------------------------------ // "tcp" "" "tcp" "" - - - - @@ -216,9 +217,12 @@ var dualStackTCPListenerTests = []struct { // TestDualStackTCPListener tests both single and double listen // to a test listener with various address families, different // listening address and same port. +// +// On DragonFly BSD, we expect the kernel version of node under test +// to be greater than or equal to 4.4. func TestDualStackTCPListener(t *testing.T) { switch runtime.GOOS { - case "dragonfly", "nacl", "plan9": // re-enable on dragonfly once the new IP control block management has landed + case "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) } if !supportsIPv4 || !supportsIPv6 { @@ -301,11 +305,14 @@ var dualStackUDPListenerTests = []struct { } // TestDualStackUDPListener tests both single and double listen -// to a test listener with various address families, differnet +// to a test listener with various address families, different // listening address and same port. +// +// On DragonFly BSD, we expect the kernel version of node under test +// to be greater than or equal to 4.4. func TestDualStackUDPListener(t *testing.T) { switch runtime.GOOS { - case "dragonfly", "nacl", "plan9": // re-enable on dragonfly once the new IP control block management has landed + case "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) } if !supportsIPv4 || !supportsIPv6 { @@ -477,13 +484,12 @@ func checkDualStackAddrFamily(fd *netFD) error { } func TestWildWildcardListener(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + switch runtime.GOOS { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } defer func() { if p := recover(); p != nil { @@ -521,12 +527,17 @@ var ipv4MulticastListenerTests = []struct { // test listener with same address family, same group address and same // port. func TestIPv4MulticastListener(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + switch runtime.GOOS { case "android", "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) case "solaris": t.Skipf("not supported on solaris, see golang.org/issue/7399") } + if !supportsIPv4 { + t.Skip("IPv4 is not supported") + } closer := func(cs []*UDPConn) { for _, c := range cs { @@ -542,7 +553,7 @@ func TestIPv4MulticastListener(t *testing.T) { // routing stuff for finding out an appropriate // nexthop containing both network and link layer // adjacencies. - if ifi == nil && (testing.Short() || !*testExternal) { + if ifi == nil || !*testIPv4 { continue } for _, tt := range ipv4MulticastListenerTests { @@ -591,6 +602,8 @@ var ipv6MulticastListenerTests = []struct { // test listener with same address family, same group address and same // port. func TestIPv6MulticastListener(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + switch runtime.GOOS { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) @@ -598,7 +611,7 @@ func TestIPv6MulticastListener(t *testing.T) { t.Skipf("not supported on solaris, see issue 7399") } if !supportsIPv6 { - t.Skip("ipv6 is not supported") + t.Skip("IPv6 is not supported") } if os.Getuid() != 0 { t.Skip("must be root") @@ -618,7 +631,7 @@ func TestIPv6MulticastListener(t *testing.T) { // routing stuff for finding out an appropriate // nexthop containing both network and link layer // adjacencies. - if ifi == nil && (testing.Short() || !*testExternal || !*testIPv6) { + if ifi == nil && !*testIPv6 { continue } for _, tt := range ipv6MulticastListenerTests { diff --git a/libgo/go/net/lookup.go b/libgo/go/net/lookup.go index 7aa111b..c169e9e 100644 --- a/libgo/go/net/lookup.go +++ b/libgo/go/net/lookup.go @@ -1,12 +1,13 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" + "internal/nettrace" "internal/singleflight" - "time" ) // protocols contains minimal mappings between internet protocol @@ -33,7 +34,7 @@ func LookupHost(host string) (addrs []string, err error) { if ip := ParseIP(host); ip != nil { return []string{host}, nil } - return lookupHost(host) + return lookupHost(context.Background(), host) } // LookupIP looks up host using the local resolver. @@ -47,7 +48,7 @@ func LookupIP(host string) (ips []IP, err error) { if ip := ParseIP(host); ip != nil { return []IP{ip}, nil } - addrs, err := lookupIPMerge(host) + addrs, err := lookupIPMerge(context.Background(), host) if err != nil { return } @@ -63,9 +64,9 @@ var lookupGroup singleflight.Group // lookupIPMerge wraps lookupIP, but makes sure that for any given // host, only one lookup is in-flight at a time. The returned memory // is always owned by the caller. -func lookupIPMerge(host string) (addrs []IPAddr, err error) { +func lookupIPMerge(ctx context.Context, host string) (addrs []IPAddr, err error) { addrsi, err, shared := lookupGroup.Do(host, func() (interface{}, error) { - return testHookLookupIP(lookupIP, host) + return testHookLookupIP(ctx, lookupIP, host) }) return lookupIPReturn(addrsi, err, shared) } @@ -85,52 +86,65 @@ func lookupIPReturn(addrsi interface{}, err error, shared bool) ([]IPAddr, error return addrs, nil } -// lookupIPDeadline looks up a hostname with a deadline. -func lookupIPDeadline(host string, deadline time.Time) (addrs []IPAddr, err error) { - if deadline.IsZero() { - return lookupIPMerge(host) +// ipAddrsEface returns an empty interface slice of addrs. +func ipAddrsEface(addrs []IPAddr) []interface{} { + s := make([]interface{}, len(addrs)) + for i, v := range addrs { + s[i] = v } + return s +} - // We could push the deadline down into the name resolution - // functions. However, the most commonly used implementation - // calls getaddrinfo, which has no timeout. - - timeout := deadline.Sub(time.Now()) - if timeout <= 0 { - return nil, errTimeout +// lookupIPContext looks up a hostname with a context. +// +// TODO(bradfitz): rename this function. All the other +// build-tag-specific lookupIP funcs also take a context now, so this +// name is no longer great. Maybe make this lookupIPMerge and ditch +// the other one, making its callers call this instead with a +// context.Background(). +func lookupIPContext(ctx context.Context, host string) (addrs []IPAddr, err error) { + trace, _ := ctx.Value(nettrace.TraceKey{}).(*nettrace.Trace) + if trace != nil && trace.DNSStart != nil { + trace.DNSStart(host) + } + // The underlying resolver func is lookupIP by default but it + // can be overridden by tests. This is needed by net/http, so it + // uses a context key instead of unexported variables. + resolverFunc := lookupIP + if alt, _ := ctx.Value(nettrace.LookupIPAltResolverKey{}).(func(context.Context, string) ([]IPAddr, error)); alt != nil { + resolverFunc = alt } - t := time.NewTimer(timeout) - defer t.Stop() ch := lookupGroup.DoChan(host, func() (interface{}, error) { - return testHookLookupIP(lookupIP, host) + return testHookLookupIP(ctx, resolverFunc, host) }) select { - case <-t.C: - // The DNS lookup timed out for some reason. Force + case <-ctx.Done(): + // The DNS lookup timed out for some reason. Force // future requests to start the DNS lookup again // rather than waiting for the current lookup to - // complete. See issue 8602. + // complete. See issue 8602. + err := mapErr(ctx.Err()) lookupGroup.Forget(host) - - return nil, errTimeout - + if trace != nil && trace.DNSDone != nil { + trace.DNSDone(nil, false, err) + } + return nil, err case r := <-ch: + if trace != nil && trace.DNSDone != nil { + addrs, _ := r.Val.([]IPAddr) + trace.DNSDone(ipAddrsEface(addrs), r.Shared, r.Err) + } return lookupIPReturn(r.Val, r.Err, r.Shared) } } // LookupPort looks up the port for the given network and service. func LookupPort(network, service string) (port int, err error) { - if service == "" { - // Lock in the legacy behavior that an empty string - // means port 0. See Issue 13610. - return 0, nil - } - port, _, ok := dtoi(service, 0) - if !ok && port != big && port != -big { - port, err = lookupPort(network, service) + port, needsLookup := parsePort(service) + if needsLookup { + port, err = lookupPort(context.Background(), network, service) if err != nil { return 0, err } @@ -146,39 +160,39 @@ func LookupPort(network, service string) (port int, err error) { // LookupHost or LookupIP directly; both take care of resolving // the canonical name as part of the lookup. func LookupCNAME(name string) (cname string, err error) { - return lookupCNAME(name) + return lookupCNAME(context.Background(), name) } // LookupSRV tries to resolve an SRV query of the given service, -// protocol, and domain name. The proto is "tcp" or "udp". +// protocol, and domain name. The proto is "tcp" or "udp". // The returned records are sorted by priority and randomized // by weight within a priority. // // LookupSRV constructs the DNS name to look up following RFC 2782. -// That is, it looks up _service._proto.name. To accommodate services +// That is, it looks up _service._proto.name. To accommodate services // publishing SRV records under non-standard names, if both service // and proto are empty strings, LookupSRV looks up name directly. func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { - return lookupSRV(service, proto, name) + return lookupSRV(context.Background(), service, proto, name) } // LookupMX returns the DNS MX records for the given domain name sorted by preference. func LookupMX(name string) (mxs []*MX, err error) { - return lookupMX(name) + return lookupMX(context.Background(), name) } // LookupNS returns the DNS NS records for the given domain name. func LookupNS(name string) (nss []*NS, err error) { - return lookupNS(name) + return lookupNS(context.Background(), name) } // LookupTXT returns the DNS TXT records for the given domain name. func LookupTXT(name string) (txts []string, err error) { - return lookupTXT(name) + return lookupTXT(context.Background(), name) } // LookupAddr performs a reverse lookup for the given address, returning a list // of names mapping to that address. func LookupAddr(addr string) (names []string, err error) { - return lookupAddr(addr) + return lookupAddr(context.Background(), addr) } diff --git a/libgo/go/net/lookup_plan9.go b/libgo/go/net/lookup_plan9.go index a331628..3f7af2a 100644 --- a/libgo/go/net/lookup_plan9.go +++ b/libgo/go/net/lookup_plan9.go @@ -1,22 +1,24 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "errors" + "io" "os" ) -func query(filename, query string, bufSize int) (res []string, err error) { +func query(ctx context.Context, filename, query string, bufSize int) (res []string, err error) { file, err := os.OpenFile(filename, os.O_RDWR, 0) if err != nil { return } defer file.Close() - _, err = file.Seek(0, 0) + _, err = file.Seek(0, io.SeekStart) if err != nil { return } @@ -24,7 +26,7 @@ func query(filename, query string, bufSize int) (res []string, err error) { if err != nil { return } - _, err = file.Seek(0, 0) + _, err = file.Seek(0, io.SeekStart) if err != nil { return } @@ -39,7 +41,7 @@ func query(filename, query string, bufSize int) (res []string, err error) { return } -func queryCS(net, host, service string) (res []string, err error) { +func queryCS(ctx context.Context, net, host, service string) (res []string, err error) { switch net { case "tcp4", "tcp6": net = "tcp" @@ -49,15 +51,15 @@ func queryCS(net, host, service string) (res []string, err error) { if host == "" { host = "*" } - return query(netdir+"/cs", net+"!"+host+"!"+service, 128) + return query(ctx, netdir+"/cs", net+"!"+host+"!"+service, 128) } -func queryCS1(net string, ip IP, port int) (clone, dest string, err error) { +func queryCS1(ctx context.Context, net string, ip IP, port int) (clone, dest string, err error) { ips := "*" if len(ip) != 0 && !ip.IsUnspecified() { ips = ip.String() } - lines, err := queryCS(net, ips, itoa(port)) + lines, err := queryCS(ctx, net, ips, itoa(port)) if err != nil { return } @@ -69,8 +71,8 @@ func queryCS1(net string, ip IP, port int) (clone, dest string, err error) { return } -func queryDNS(addr string, typ string) (res []string, err error) { - return query(netdir+"/dns", addr+" "+typ, 1024) +func queryDNS(ctx context.Context, addr string, typ string) (res []string, err error) { + return query(ctx, netdir+"/dns", addr+" "+typ, 1024) } // toLower returns a lower-case version of in. Restricting us to @@ -96,8 +98,8 @@ func toLower(in string) string { // lookupProtocol looks up IP protocol name and returns // the corresponding protocol number. -func lookupProtocol(name string) (proto int, err error) { - lines, err := query(netdir+"/cs", "!protocol="+toLower(name), 128) +func lookupProtocol(ctx context.Context, name string) (proto int, err error) { + lines, err := query(ctx, netdir+"/cs", "!protocol="+toLower(name), 128) if err != nil { return 0, err } @@ -115,10 +117,10 @@ func lookupProtocol(name string) (proto int, err error) { return 0, UnknownNetworkError(name) } -func lookupHost(host string) (addrs []string, err error) { +func lookupHost(ctx context.Context, host string) (addrs []string, err error) { // Use netdir/cs instead of netdir/dns because cs knows about // host names in local network (e.g. from /lib/ndb/local) - lines, err := queryCS("net", host, "1") + lines, err := queryCS(ctx, "net", host, "1") if err != nil { return } @@ -146,8 +148,8 @@ loop: return } -func lookupIP(host string) (addrs []IPAddr, err error) { - lits, err := LookupHost(host) +func lookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { + lits, err := lookupHost(ctx, host) if err != nil { return } @@ -161,14 +163,14 @@ func lookupIP(host string) (addrs []IPAddr, err error) { return } -func lookupPort(network, service string) (port int, err error) { +func lookupPort(ctx context.Context, network, service string) (port int, err error) { switch network { case "tcp4", "tcp6": network = "tcp" case "udp4", "udp6": network = "udp" } - lines, err := queryCS(network, "127.0.0.1", service) + lines, err := queryCS(ctx, network, "127.0.0.1", service) if err != nil { return } @@ -190,8 +192,8 @@ func lookupPort(network, service string) (port int, err error) { return 0, unknownPortError } -func lookupCNAME(name string) (cname string, err error) { - lines, err := queryDNS(name, "cname") +func lookupCNAME(ctx context.Context, name string) (cname string, err error) { + lines, err := queryDNS(ctx, name, "cname") if err != nil { return } @@ -203,14 +205,14 @@ func lookupCNAME(name string) (cname string, err error) { return "", errors.New("bad response from ndb/dns") } -func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { +func lookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*SRV, err error) { var target string if service == "" && proto == "" { target = name } else { target = "_" + service + "._" + proto + "." + name } - lines, err := queryDNS(target, "srv") + lines, err := queryDNS(ctx, target, "srv") if err != nil { return } @@ -232,8 +234,8 @@ func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err err return } -func lookupMX(name string) (mx []*MX, err error) { - lines, err := queryDNS(name, "mx") +func lookupMX(ctx context.Context, name string) (mx []*MX, err error) { + lines, err := queryDNS(ctx, name, "mx") if err != nil { return } @@ -250,8 +252,8 @@ func lookupMX(name string) (mx []*MX, err error) { return } -func lookupNS(name string) (ns []*NS, err error) { - lines, err := queryDNS(name, "ns") +func lookupNS(ctx context.Context, name string) (ns []*NS, err error) { + lines, err := queryDNS(ctx, name, "ns") if err != nil { return } @@ -265,8 +267,8 @@ func lookupNS(name string) (ns []*NS, err error) { return } -func lookupTXT(name string) (txt []string, err error) { - lines, err := queryDNS(name, "txt") +func lookupTXT(ctx context.Context, name string) (txt []string, err error) { + lines, err := queryDNS(ctx, name, "txt") if err != nil { return } @@ -278,12 +280,12 @@ func lookupTXT(name string) (txt []string, err error) { return } -func lookupAddr(addr string) (name []string, err error) { +func lookupAddr(ctx context.Context, addr string) (name []string, err error) { arpa, err := reverseaddr(addr) if err != nil { return } - lines, err := queryDNS(arpa, "ptr") + lines, err := queryDNS(ctx, arpa, "ptr") if err != nil { return } diff --git a/libgo/go/net/lookup_stub.go b/libgo/go/net/lookup_stub.go index 5636198..bd096b3 100644 --- a/libgo/go/net/lookup_stub.go +++ b/libgo/go/net/lookup_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -6,44 +6,47 @@ package net -import "syscall" +import ( + "context" + "syscall" +) -func lookupProtocol(name string) (proto int, err error) { +func lookupProtocol(ctx context.Context, name string) (proto int, err error) { return 0, syscall.ENOPROTOOPT } -func lookupHost(host string) (addrs []string, err error) { +func lookupHost(ctx context.Context, host string) (addrs []string, err error) { return nil, syscall.ENOPROTOOPT } -func lookupIP(host string) (addrs []IPAddr, err error) { +func lookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { return nil, syscall.ENOPROTOOPT } -func lookupPort(network, service string) (port int, err error) { +func lookupPort(ctx context.Context, network, service string) (port int, err error) { return 0, syscall.ENOPROTOOPT } -func lookupCNAME(name string) (cname string, err error) { +func lookupCNAME(ctx context.Context, name string) (cname string, err error) { return "", syscall.ENOPROTOOPT } -func lookupSRV(service, proto, name string) (cname string, srvs []*SRV, err error) { +func lookupSRV(ctx context.Context, service, proto, name string) (cname string, srvs []*SRV, err error) { return "", nil, syscall.ENOPROTOOPT } -func lookupMX(name string) (mxs []*MX, err error) { +func lookupMX(ctx context.Context, name string) (mxs []*MX, err error) { return nil, syscall.ENOPROTOOPT } -func lookupNS(name string) (nss []*NS, err error) { +func lookupNS(ctx context.Context, name string) (nss []*NS, err error) { return nil, syscall.ENOPROTOOPT } -func lookupTXT(name string) (txts []string, err error) { +func lookupTXT(ctx context.Context, name string) (txts []string, err error) { return nil, syscall.ENOPROTOOPT } -func lookupAddr(addr string) (ptrs []string, err error) { +func lookupAddr(ctx context.Context, addr string) (ptrs []string, err error) { return nil, syscall.ENOPROTOOPT } diff --git a/libgo/go/net/lookup_test.go b/libgo/go/net/lookup_test.go index 439496a..b3aeb85 100644 --- a/libgo/go/net/lookup_test.go +++ b/libgo/go/net/lookup_test.go @@ -6,6 +6,7 @@ package net import ( "bytes" + "context" "fmt" "internal/testenv" "runtime" @@ -14,7 +15,7 @@ import ( "time" ) -func lookupLocalhost(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { +func lookupLocalhost(ctx context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { switch host { case "localhost": return []IPAddr{ @@ -22,7 +23,7 @@ func lookupLocalhost(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, {IP: IPv6loopback}, }, nil default: - return fn(host) + return fn(ctx, host) } } @@ -58,9 +59,10 @@ var lookupGoogleSRVTests = []struct { } func TestLookupGoogleSRV(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -68,6 +70,7 @@ func TestLookupGoogleSRV(t *testing.T) { for _, tt := range lookupGoogleSRVTests { cname, srvs, err := LookupSRV(tt.service, tt.proto, tt.name) if err != nil { + testenv.SkipFlakyNet(t) t.Fatal(err) } if len(srvs) == 0 { @@ -92,9 +95,10 @@ var lookupGmailMXTests = []struct { } func TestLookupGmailMX(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -123,9 +127,10 @@ var lookupGmailNSTests = []struct { } func TestLookupGmailNS(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -133,6 +138,7 @@ func TestLookupGmailNS(t *testing.T) { for _, tt := range lookupGmailNSTests { nss, err := LookupNS(tt.name) if err != nil { + testenv.SkipFlakyNet(t) t.Fatal(err) } if len(nss) == 0 { @@ -154,9 +160,10 @@ var lookupGmailTXTTests = []struct { } func TestLookupGmailTXT(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -188,9 +195,10 @@ var lookupGooglePublicDNSAddrTests = []struct { } func TestLookupGooglePublicDNSAddr(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !supportsIPv6 || !*testIPv4 || !*testIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -243,9 +251,10 @@ var lookupIANACNAMETests = []struct { } func TestLookupIANACNAME(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -269,9 +278,10 @@ var lookupGoogleHostTests = []struct { } func TestLookupGoogleHost(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -300,9 +310,10 @@ var lookupGoogleIPTests = []struct { } func TestLookupGoogleIP(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -360,22 +371,38 @@ func TestReverseAddress(t *testing.T) { } } -func TestLookupIPDeadline(t *testing.T) { +func TestDNSFlood(t *testing.T) { if !*testDNSFlood { t.Skip("test disabled; use -dnsflood to enable") } - const N = 5000 + var N = 5000 + if runtime.GOOS == "darwin" { + // On Darwin this test consumes kernel threads much + // than other platforms for some reason. + // When we monitor the number of allocated Ms by + // observing on runtime.newm calls, we can see that it + // easily reaches the per process ceiling + // kern.num_threads when CGO_ENABLED=1 and + // GODEBUG=netdns=go. + N = 500 + } + const timeout = 3 * time.Second + ctxHalfTimeout, cancel := context.WithTimeout(context.Background(), timeout/2) + defer cancel() + ctxTimeout, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + c := make(chan error, 2*N) for i := 0; i < N; i++ { name := fmt.Sprintf("%d.net-test.golang.org", i) go func() { - _, err := lookupIPDeadline(name, time.Now().Add(timeout/2)) + _, err := lookupIPContext(ctxHalfTimeout, name) c <- err }() go func() { - _, err := lookupIPDeadline(name, time.Now().Add(timeout)) + _, err := lookupIPContext(ctxTimeout, name) c <- err }() } @@ -426,6 +453,10 @@ func TestLookupDotsWithLocalSource(t *testing.T) { t.Skip("IPv4 is required") } + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) + } + for i, fn := range []func() func(){forceGoDNS, forceCgoDNS} { fixup := fn() if fixup == nil { @@ -463,9 +494,10 @@ func TestLookupDotsWithLocalSource(t *testing.T) { } func TestLookupDotsWithRemoteSource(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -483,6 +515,7 @@ func TestLookupDotsWithRemoteSource(t *testing.T) { func testDots(t *testing.T, mode string) { names, err := LookupAddr("8.8.8.8") // Google dns server if err != nil { + testenv.SkipFlakyNet(t) t.Errorf("LookupAddr(8.8.8.8): %v (mode=%v)", err, mode) } else { for _, name := range names { @@ -494,12 +527,16 @@ func testDots(t *testing.T, mode string) { } cname, err := LookupCNAME("www.mit.edu") - if err != nil || !strings.HasSuffix(cname, ".") { - t.Errorf("LookupCNAME(www.mit.edu) = %v, %v, want cname ending in . with trailing dot (mode=%v)", cname, err, mode) + if err != nil { + testenv.SkipFlakyNet(t) + t.Errorf("LookupCNAME(www.mit.edu, mode=%v): %v", mode, err) + } else if !strings.HasSuffix(cname, ".") { + t.Errorf("LookupCNAME(www.mit.edu) = %v, want cname ending in . with trailing dot (mode=%v)", cname, mode) } mxs, err := LookupMX("google.com") if err != nil { + testenv.SkipFlakyNet(t) t.Errorf("LookupMX(google.com): %v (mode=%v)", err, mode) } else { for _, mx := range mxs { @@ -512,6 +549,7 @@ func testDots(t *testing.T, mode string) { nss, err := LookupNS("google.com") if err != nil { + testenv.SkipFlakyNet(t) t.Errorf("LookupNS(google.com): %v (mode=%v)", err, mode) } else { for _, ns := range nss { @@ -524,6 +562,7 @@ func testDots(t *testing.T, mode string) { cname, srvs, err := LookupSRV("xmpp-server", "tcp", "google.com") if err != nil { + testenv.SkipFlakyNet(t) t.Errorf("LookupSRV(xmpp-server, tcp, google.com): %v (mode=%v)", err, mode) } else { if !strings.HasSuffix(cname, ".google.com.") { @@ -574,61 +613,60 @@ func srvString(srvs []*SRV) string { return buf.String() } -var lookupPortTests = []struct { - network string - name string - port int - ok bool -}{ - {"tcp", "0", 0, true}, - {"tcp", "echo", 7, true}, - {"tcp", "discard", 9, true}, - {"tcp", "systat", 11, true}, - {"tcp", "daytime", 13, true}, - {"tcp", "chargen", 19, true}, - {"tcp", "ftp-data", 20, true}, - {"tcp", "ftp", 21, true}, - {"tcp", "telnet", 23, true}, - {"tcp", "smtp", 25, true}, - {"tcp", "time", 37, true}, - {"tcp", "domain", 53, true}, - {"tcp", "finger", 79, true}, - {"tcp", "42", 42, true}, - - {"udp", "0", 0, true}, - {"udp", "echo", 7, true}, - {"udp", "tftp", 69, true}, - {"udp", "bootpc", 68, true}, - {"udp", "bootps", 67, true}, - {"udp", "domain", 53, true}, - {"udp", "ntp", 123, true}, - {"udp", "snmp", 161, true}, - {"udp", "syslog", 514, true}, - {"udp", "42", 42, true}, - - {"--badnet--", "zzz", 0, false}, - {"tcp", "--badport--", 0, false}, - {"tcp", "-1", 0, false}, - {"tcp", "65536", 0, false}, - {"udp", "-1", 0, false}, - {"udp", "65536", 0, false}, - - // Issue 13610: LookupPort("tcp", "") - {"tcp", "", 0, true}, - {"tcp6", "", 0, true}, - {"tcp4", "", 0, true}, - {"udp", "", 0, true}, -} - func TestLookupPort(t *testing.T) { + // See http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml + // + // Please be careful about adding new mappings for testings. + // There are platforms having incomplete mappings for + // restricted resource access and security reasons. + type test struct { + network string + name string + port int + ok bool + } + var tests = []test{ + {"tcp", "0", 0, true}, + {"udp", "0", 0, true}, + {"udp", "domain", 53, true}, + + {"--badnet--", "zzz", 0, false}, + {"tcp", "--badport--", 0, false}, + {"tcp", "-1", 0, false}, + {"tcp", "65536", 0, false}, + {"udp", "-1", 0, false}, + {"udp", "65536", 0, false}, + {"tcp", "123456789", 0, false}, + + // Issue 13610: LookupPort("tcp", "") + {"tcp", "", 0, true}, + {"tcp4", "", 0, true}, + {"tcp6", "", 0, true}, + {"udp", "", 0, true}, + {"udp4", "", 0, true}, + {"udp6", "", 0, true}, + } + switch runtime.GOOS { case "nacl": t.Skipf("not supported on %s", runtime.GOOS) + case "android": + if netGo { + t.Skipf("not supported on %s without cgo; see golang.org/issues/14576", runtime.GOOS) + } + default: + tests = append(tests, test{"tcp", "http", 80, true}) } - for _, tt := range lookupPortTests { - if port, err := LookupPort(tt.network, tt.name); port != tt.port || (err == nil) != tt.ok { - t.Errorf("LookupPort(%q, %q) = %d, %v; want %d", tt.network, tt.name, port, err, tt.port) + for _, tt := range tests { + port, err := LookupPort(tt.network, tt.name) + if port != tt.port || (err == nil) != tt.ok { + t.Errorf("LookupPort(%q, %q) = %d, %v; want %d, error=%t", tt.network, tt.name, port, err, tt.port, !tt.ok) + } + if err != nil { + if perr := parseLookupPortError(err); perr != nil { + t.Error(perr) + } } } } diff --git a/libgo/go/net/lookup_unix.go b/libgo/go/net/lookup_unix.go index a64da8b..15397e8 100644 --- a/libgo/go/net/lookup_unix.go +++ b/libgo/go/net/lookup_unix.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -6,7 +6,10 @@ package net -import "sync" +import ( + "context" + "sync" +) var onceReadProtocols sync.Once @@ -40,7 +43,7 @@ func readProtocols() { // lookupProtocol looks up IP protocol name in /etc/protocols and // returns correspondent protocol number. -func lookupProtocol(name string) (int, error) { +func lookupProtocol(_ context.Context, name string) (int, error) { onceReadProtocols.Do(readProtocols) proto, found := protocols[name] if !found { @@ -49,56 +52,61 @@ func lookupProtocol(name string) (int, error) { return proto, nil } -func lookupHost(host string) (addrs []string, err error) { +func lookupHost(ctx context.Context, host string) (addrs []string, err error) { order := systemConf().hostLookupOrder(host) if order == hostLookupCgo { - if addrs, err, ok := cgoLookupHost(host); ok { + if addrs, err, ok := cgoLookupHost(ctx, host); ok { return addrs, err } // cgo not available (or netgo); fall back to Go's DNS resolver order = hostLookupFilesDNS } - return goLookupHostOrder(host, order) + return goLookupHostOrder(ctx, host, order) } -func lookupIP(host string) (addrs []IPAddr, err error) { +func lookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { order := systemConf().hostLookupOrder(host) if order == hostLookupCgo { - if addrs, err, ok := cgoLookupIP(host); ok { + if addrs, err, ok := cgoLookupIP(ctx, host); ok { return addrs, err } // cgo not available (or netgo); fall back to Go's DNS resolver order = hostLookupFilesDNS } - return goLookupIPOrder(host, order) + return goLookupIPOrder(ctx, host, order) } -func lookupPort(network, service string) (int, error) { +func lookupPort(ctx context.Context, network, service string) (int, error) { + // TODO: use the context if there ever becomes a need. Related + // is issue 15321. But port lookup generally just involves + // local files, and the os package has no context support. The + // files might be on a remote filesystem, though. This should + // probably race goroutines if ctx != context.Background(). if systemConf().canUseCgo() { - if port, err, ok := cgoLookupPort(network, service); ok { + if port, err, ok := cgoLookupPort(ctx, network, service); ok { return port, err } } return goLookupPort(network, service) } -func lookupCNAME(name string) (string, error) { +func lookupCNAME(ctx context.Context, name string) (string, error) { if systemConf().canUseCgo() { - if cname, err, ok := cgoLookupCNAME(name); ok { + if cname, err, ok := cgoLookupCNAME(ctx, name); ok { return cname, err } } - return goLookupCNAME(name) + return goLookupCNAME(ctx, name) } -func lookupSRV(service, proto, name string) (string, []*SRV, error) { +func lookupSRV(ctx context.Context, service, proto, name string) (string, []*SRV, error) { var target string if service == "" && proto == "" { target = name } else { target = "_" + service + "._" + proto + "." + name } - cname, rrs, err := lookup(target, dnsTypeSRV) + cname, rrs, err := lookup(ctx, target, dnsTypeSRV) if err != nil { return "", nil, err } @@ -111,8 +119,8 @@ func lookupSRV(service, proto, name string) (string, []*SRV, error) { return cname, srvs, nil } -func lookupMX(name string) ([]*MX, error) { - _, rrs, err := lookup(name, dnsTypeMX) +func lookupMX(ctx context.Context, name string) ([]*MX, error) { + _, rrs, err := lookup(ctx, name, dnsTypeMX) if err != nil { return nil, err } @@ -125,8 +133,8 @@ func lookupMX(name string) ([]*MX, error) { return mxs, nil } -func lookupNS(name string) ([]*NS, error) { - _, rrs, err := lookup(name, dnsTypeNS) +func lookupNS(ctx context.Context, name string) ([]*NS, error) { + _, rrs, err := lookup(ctx, name, dnsTypeNS) if err != nil { return nil, err } @@ -137,8 +145,8 @@ func lookupNS(name string) ([]*NS, error) { return nss, nil } -func lookupTXT(name string) ([]string, error) { - _, rrs, err := lookup(name, dnsTypeTXT) +func lookupTXT(ctx context.Context, name string) ([]string, error) { + _, rrs, err := lookup(ctx, name, dnsTypeTXT) if err != nil { return nil, err } @@ -149,11 +157,11 @@ func lookupTXT(name string) ([]string, error) { return txts, nil } -func lookupAddr(addr string) ([]string, error) { +func lookupAddr(ctx context.Context, addr string) ([]string, error) { if systemConf().canUseCgo() { - if ptrs, err, ok := cgoLookupPTR(addr); ok { + if ptrs, err, ok := cgoLookupPTR(ctx, addr); ok { return ptrs, err } } - return goLookupPTR(addr) + return goLookupPTR(ctx, addr) } diff --git a/libgo/go/net/lookup_windows.go b/libgo/go/net/lookup_windows.go index 13edc26..5f65c2d 100644 --- a/libgo/go/net/lookup_windows.go +++ b/libgo/go/net/lookup_windows.go @@ -5,17 +5,13 @@ package net import ( + "context" "os" "runtime" "syscall" "unsafe" ) -var ( - lookupPort = oldLookupPort - lookupIP = oldLookupIP -) - func getprotobyname(name string) (proto int, err error) { p, err := syscall.GetProtoByName(name) if err != nil { @@ -25,34 +21,41 @@ func getprotobyname(name string) (proto int, err error) { } // lookupProtocol looks up IP protocol name and returns correspondent protocol number. -func lookupProtocol(name string) (int, error) { +func lookupProtocol(ctx context.Context, name string) (int, error) { // GetProtoByName return value is stored in thread local storage. // Start new os thread before the call to prevent races. type result struct { proto int err error } - ch := make(chan result) + ch := make(chan result) // unbuffered go func() { acquireThread() defer releaseThread() runtime.LockOSThread() defer runtime.UnlockOSThread() proto, err := getprotobyname(name) - ch <- result{proto: proto, err: err} + select { + case ch <- result{proto: proto, err: err}: + case <-ctx.Done(): + } }() - r := <-ch - if r.err != nil { - if proto, ok := protocols[name]; ok { - return proto, nil + select { + case r := <-ch: + if r.err != nil { + if proto, ok := protocols[name]; ok { + return proto, nil + } + r.err = &DNSError{Err: r.err.Error(), Name: name} } - r.err = &DNSError{Err: r.err.Error(), Name: name} + return r.proto, r.err + case <-ctx.Done(): + return 0, mapErr(ctx.Err()) } - return r.proto, r.err } -func lookupHost(name string) ([]string, error) { - ips, err := LookupIP(name) +func lookupHost(ctx context.Context, name string) ([]string, error) { + ips, err := lookupIP(ctx, name) if err != nil { return nil, err } @@ -63,121 +66,67 @@ func lookupHost(name string) ([]string, error) { return addrs, nil } -func gethostbyname(name string) (addrs []IPAddr, err error) { - // caller already acquired thread - h, err := syscall.GetHostByName(name) - if err != nil { - return nil, os.NewSyscallError("gethostbyname", err) - } - switch h.AddrType { - case syscall.AF_INET: - i := 0 - addrs = make([]IPAddr, 100) // plenty of room to grow - for p := (*[100](*[4]byte))(unsafe.Pointer(h.AddrList)); i < cap(addrs) && p[i] != nil; i++ { - addrs[i] = IPAddr{IP: IPv4(p[i][0], p[i][1], p[i][2], p[i][3])} - } - addrs = addrs[0:i] - default: // TODO(vcc): Implement non IPv4 address lookups. - return nil, syscall.EWINDOWS - } - return addrs, nil -} +func lookupIP(ctx context.Context, name string) ([]IPAddr, error) { + // TODO(bradfitz,brainman): use ctx? -func oldLookupIP(name string) ([]IPAddr, error) { - // GetHostByName return value is stored in thread local storage. - // Start new os thread before the call to prevent races. - type result struct { + type ret struct { addrs []IPAddr err error } - ch := make(chan result) + ch := make(chan ret, 1) go func() { acquireThread() defer releaseThread() - runtime.LockOSThread() - defer runtime.UnlockOSThread() - addrs, err := gethostbyname(name) - ch <- result{addrs: addrs, err: err} - }() - r := <-ch - if r.err != nil { - r.err = &DNSError{Err: r.err.Error(), Name: name} - } - return r.addrs, r.err -} - -func newLookupIP(name string) ([]IPAddr, error) { - acquireThread() - defer releaseThread() - hints := syscall.AddrinfoW{ - Family: syscall.AF_UNSPEC, - Socktype: syscall.SOCK_STREAM, - Protocol: syscall.IPPROTO_IP, - } - var result *syscall.AddrinfoW - e := syscall.GetAddrInfoW(syscall.StringToUTF16Ptr(name), nil, &hints, &result) - if e != nil { - return nil, &DNSError{Err: os.NewSyscallError("getaddrinfow", e).Error(), Name: name} - } - defer syscall.FreeAddrInfoW(result) - addrs := make([]IPAddr, 0, 5) - for ; result != nil; result = result.Next { - addr := unsafe.Pointer(result.Addr) - switch result.Family { - case syscall.AF_INET: - a := (*syscall.RawSockaddrInet4)(addr).Addr - addrs = append(addrs, IPAddr{IP: IPv4(a[0], a[1], a[2], a[3])}) - case syscall.AF_INET6: - a := (*syscall.RawSockaddrInet6)(addr).Addr - zone := zoneToString(int((*syscall.RawSockaddrInet6)(addr).Scope_id)) - addrs = append(addrs, IPAddr{IP: IP{a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]}, Zone: zone}) - default: - return nil, &DNSError{Err: syscall.EWINDOWS.Error(), Name: name} + hints := syscall.AddrinfoW{ + Family: syscall.AF_UNSPEC, + Socktype: syscall.SOCK_STREAM, + Protocol: syscall.IPPROTO_IP, } - } - return addrs, nil -} - -func getservbyname(network, service string) (int, error) { - acquireThread() - defer releaseThread() - switch network { - case "tcp4", "tcp6": - network = "tcp" - case "udp4", "udp6": - network = "udp" - } - s, err := syscall.GetServByName(service, network) - if err != nil { - return 0, os.NewSyscallError("getservbyname", err) - } - return int(syscall.Ntohs(s.Port)), nil -} - -func oldLookupPort(network, service string) (int, error) { - // GetServByName return value is stored in thread local storage. - // Start new os thread before the call to prevent races. - type result struct { - port int - err error - } - ch := make(chan result) - go func() { - acquireThread() - defer releaseThread() - runtime.LockOSThread() - defer runtime.UnlockOSThread() - port, err := getservbyname(network, service) - ch <- result{port: port, err: err} + var result *syscall.AddrinfoW + e := syscall.GetAddrInfoW(syscall.StringToUTF16Ptr(name), nil, &hints, &result) + if e != nil { + ch <- ret{err: &DNSError{Err: os.NewSyscallError("getaddrinfow", e).Error(), Name: name}} + } + defer syscall.FreeAddrInfoW(result) + addrs := make([]IPAddr, 0, 5) + for ; result != nil; result = result.Next { + addr := unsafe.Pointer(result.Addr) + switch result.Family { + case syscall.AF_INET: + a := (*syscall.RawSockaddrInet4)(addr).Addr + addrs = append(addrs, IPAddr{IP: IPv4(a[0], a[1], a[2], a[3])}) + case syscall.AF_INET6: + a := (*syscall.RawSockaddrInet6)(addr).Addr + zone := zoneToString(int((*syscall.RawSockaddrInet6)(addr).Scope_id)) + addrs = append(addrs, IPAddr{IP: IP{a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]}, Zone: zone}) + default: + ch <- ret{err: &DNSError{Err: syscall.EWINDOWS.Error(), Name: name}} + } + } + ch <- ret{addrs: addrs} }() - r := <-ch - if r.err != nil { - r.err = &DNSError{Err: r.err.Error(), Name: network + "/" + service} + select { + case r := <-ch: + return r.addrs, r.err + case <-ctx.Done(): + // TODO(bradfitz,brainman): cancel the ongoing + // GetAddrInfoW? It would require conditionally using + // GetAddrInfoEx with lpOverlapped, which requires + // Windows 8 or newer. I guess we'll need oldLookupIP, + // newLookupIP, and newerLookUP. + // + // For now we just let it finish and write to the + // buffered channel. + return nil, &DNSError{ + Name: name, + Err: ctx.Err().Error(), + IsTimeout: ctx.Err() == context.DeadlineExceeded, + } } - return r.port, r.err } -func newLookupPort(network, service string) (int, error) { +func lookupPort(ctx context.Context, network, service string) (int, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var stype int32 @@ -213,7 +162,8 @@ func newLookupPort(network, service string) (int, error) { return 0, &DNSError{Err: syscall.EINVAL.Error(), Name: network + "/" + service} } -func lookupCNAME(name string) (string, error) { +func lookupCNAME(ctx context.Context, name string) (string, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var r *syscall.DNSRecord @@ -233,7 +183,8 @@ func lookupCNAME(name string) (string, error) { return absDomainName([]byte(cname)), nil } -func lookupSRV(service, proto, name string) (string, []*SRV, error) { +func lookupSRV(ctx context.Context, service, proto, name string) (string, []*SRV, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var target string @@ -258,7 +209,8 @@ func lookupSRV(service, proto, name string) (string, []*SRV, error) { return absDomainName([]byte(target)), srvs, nil } -func lookupMX(name string) ([]*MX, error) { +func lookupMX(ctx context.Context, name string) ([]*MX, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var r *syscall.DNSRecord @@ -277,7 +229,8 @@ func lookupMX(name string) ([]*MX, error) { return mxs, nil } -func lookupNS(name string) ([]*NS, error) { +func lookupNS(ctx context.Context, name string) ([]*NS, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var r *syscall.DNSRecord @@ -295,7 +248,8 @@ func lookupNS(name string) ([]*NS, error) { return nss, nil } -func lookupTXT(name string) ([]string, error) { +func lookupTXT(ctx context.Context, name string) ([]string, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var r *syscall.DNSRecord @@ -316,7 +270,8 @@ func lookupTXT(name string) ([]string, error) { return txts, nil } -func lookupAddr(addr string) ([]string, error) { +func lookupAddr(ctx context.Context, addr string) ([]string, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() arpa, err := reverseaddr(addr) diff --git a/libgo/go/net/mac.go b/libgo/go/net/mac.go index 93f0b09..f3b1694 100644 --- a/libgo/go/net/mac.go +++ b/libgo/go/net/mac.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/mac_test.go b/libgo/go/net/mac_test.go index 1ec6b28..2630d19 100644 --- a/libgo/go/net/mac_test.go +++ b/libgo/go/net/mac_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/mail/message.go b/libgo/go/net/mail/message.go index 923630c..0c00069 100644 --- a/libgo/go/net/mail/message.go +++ b/libgo/go/net/mail/message.go @@ -5,13 +5,15 @@ /* Package mail implements parsing of mail messages. -For the most part, this package follows the syntax as specified by RFC 5322. +For the most part, this package follows the syntax as specified by RFC 5322 and +extended by RFC 6532. Notable divergences: * Obsolete address formats are not parsed, including addresses with embedded route information. * Group addresses are not parsed. * The full range of spacing (the CFWS syntax element) is not supported, such as breaking addresses across lines. + * No unicode normalization is performed. */ package mail @@ -26,6 +28,7 @@ import ( "net/textproto" "strings" "time" + "unicode/utf8" ) var debug = debugT(false) @@ -138,7 +141,7 @@ type Address struct { // Parses a single RFC 5322 address, e.g. "Barry Gibbs <bg@example.com>" func ParseAddress(address string) (*Address, error) { - return (&addrParser{s: address}).parseAddress() + return (&addrParser{s: address}).parseSingleAddress() } // ParseAddressList parses the given string as a list of addresses. @@ -155,7 +158,7 @@ type AddressParser struct { // Parse parses a single RFC 5322 address of the // form "Gogh Fir <gf@example.com>" or "foo@example.com". func (p *AddressParser) Parse(address string) (*Address, error) { - return (&addrParser{s: address, dec: p.WordDecoder}).parseAddress() + return (&addrParser{s: address, dec: p.WordDecoder}).parseSingleAddress() } // ParseList parses the given string as a list of comma-separated addresses @@ -168,7 +171,6 @@ func (p *AddressParser) ParseList(list string) ([]*Address, error) { // If the address's name contains non-ASCII characters // the name will be rendered according to RFC 2047. func (a *Address) String() string { - // Format address local@domain at := strings.LastIndex(a.Address, "@") var local, domain string @@ -181,15 +183,12 @@ func (a *Address) String() string { } // Add quotes if needed - // TODO: rendering quoted local part and rendering printable name - // should be merged in helper function. quoteLocal := false - for i := 0; i < len(local); i++ { - ch := local[i] - if isAtext(ch, false) { + for i, r := range local { + if isAtext(r, false) { continue } - if ch == '.' { + if r == '.' { // Dots are okay if they are surrounded by atext. // We only need to check that the previous byte is // not a dot, and this isn't the end of the string. @@ -213,25 +212,16 @@ func (a *Address) String() string { // If every character is printable ASCII, quoting is simple. allPrintable := true - for i := 0; i < len(a.Name); i++ { + for _, r := range a.Name { // isWSP here should actually be isFWS, // but we don't support folding yet. - if !isVchar(a.Name[i]) && !isWSP(a.Name[i]) { + if !isVchar(r) && !isWSP(r) || isMultibyte(r) { allPrintable = false break } } if allPrintable { - b := bytes.NewBufferString(`"`) - for i := 0; i < len(a.Name); i++ { - if !isQtext(a.Name[i]) && !isWSP(a.Name[i]) { - b.WriteByte('\\') - } - b.WriteByte(a.Name[i]) - } - b.WriteString(`" `) - b.WriteString(s) - return b.String() + return quoteString(a.Name) + " " + s } // Text in an encoded-word in a display-name must not contain certain @@ -269,6 +259,18 @@ func (p *addrParser) parseAddressList() ([]*Address, error) { return list, nil } +func (p *addrParser) parseSingleAddress() (*Address, error) { + addr, err := p.parseAddress() + if err != nil { + return nil, err + } + p.skipSpace() + if !p.empty() { + return nil, fmt.Errorf("mail: expected single address, got %q", p.s) + } + return addr, nil +} + // parseAddress parses a single RFC 5322 address at the start of p. func (p *addrParser) parseAddress() (addr *Address, err error) { debug.Printf("parseAddress: %q", p.s) @@ -416,29 +418,48 @@ func (p *addrParser) consumePhrase() (phrase string, err error) { func (p *addrParser) consumeQuotedString() (qs string, err error) { // Assume first byte is '"'. i := 1 - qsb := make([]byte, 0, 10) + qsb := make([]rune, 0, 10) + + escaped := false + Loop: for { - if i >= p.len() { + r, size := utf8.DecodeRuneInString(p.s[i:]) + + switch { + case size == 0: return "", errors.New("mail: unclosed quoted-string") - } - switch c := p.s[i]; { - case c == '"': - break Loop - case c == '\\': - if i+1 == p.len() { - return "", errors.New("mail: unclosed quoted-string") + + case size == 1 && r == utf8.RuneError: + return "", fmt.Errorf("mail: invalid utf-8 in quoted-string: %q", p.s) + + case escaped: + // quoted-pair = ("\" (VCHAR / WSP)) + + if !isVchar(r) && !isWSP(r) { + return "", fmt.Errorf("mail: bad character in quoted-string: %q", r) } - qsb = append(qsb, p.s[i+1]) - i += 2 - case isQtext(c), c == ' ': + + qsb = append(qsb, r) + escaped = false + + case isQtext(r) || isWSP(r): // qtext (printable US-ASCII excluding " and \), or // FWS (almost; we're ignoring CRLF) - qsb = append(qsb, c) - i++ + qsb = append(qsb, r) + + case r == '"': + break Loop + + case r == '\\': + escaped = true + default: - return "", fmt.Errorf("mail: bad character in quoted-string: %q", c) + return "", fmt.Errorf("mail: bad character in quoted-string: %q", r) + } + + i += size } p.s = p.s[i+1:] if len(qsb) == 0 { @@ -447,26 +468,34 @@ Loop: return string(qsb), nil } -var errNonASCII = errors.New("mail: unencoded non-ASCII text in address") - // consumeAtom parses an RFC 5322 atom at the start of p. // If dot is true, consumeAtom parses an RFC 5322 dot-atom instead. // If permissive is true, consumeAtom will not fail on // leading/trailing/double dots in the atom (see golang.org/issue/4938). func (p *addrParser) consumeAtom(dot bool, permissive bool) (atom string, err error) { - if c := p.peek(); !isAtext(c, false) { - if c > 127 { - return "", errNonASCII + i := 0 + +Loop: + for { + r, size := utf8.DecodeRuneInString(p.s[i:]) + + switch { + case size == 1 && r == utf8.RuneError: + return "", fmt.Errorf("mail: invalid utf-8 in address: %q", p.s) + + case size == 0 || !isAtext(r, dot): + break Loop + + default: + i += size + } - return "", errors.New("mail: invalid string") - } - i := 1 - for ; i < p.len() && isAtext(p.s[i], dot); i++ { } - if i < p.len() && p.s[i] > 127 { - return "", errNonASCII + + if i == 0 { + return "", errors.New("mail: invalid string") } - atom, p.s = string(p.s[:i]), p.s[i:] + atom, p.s = p.s[:i], p.s[i:] if !permissive { if strings.HasPrefix(atom, ".") { return "", errors.New("mail: leading dot in atom") @@ -536,54 +565,58 @@ func (e charsetError) Error() string { return fmt.Sprintf("charset not supported: %q", string(e)) } -var atextChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ" + - "abcdefghijklmnopqrstuvwxyz" + - "0123456789" + - "!#$%&'*+-/=?^_`{|}~") - -// isAtext reports whether c is an RFC 5322 atext character. +// isAtext reports whether r is an RFC 5322 atext character. // If dot is true, period is included. -func isAtext(c byte, dot bool) bool { - if dot && c == '.' { - return true +func isAtext(r rune, dot bool) bool { + switch r { + case '.': + return dot + + case '(', ')', '<', '>', '[', ']', ':', ';', '@', '\\', ',', '"': // RFC 5322 3.2.3. specials + return false } - return bytes.IndexByte(atextChars, c) >= 0 + return isVchar(r) } -// isQtext reports whether c is an RFC 5322 qtext character. -func isQtext(c byte) bool { +// isQtext reports whether r is an RFC 5322 qtext character. +func isQtext(r rune) bool { // Printable US-ASCII, excluding backslash or quote. - if c == '\\' || c == '"' { + if r == '\\' || r == '"' { return false } - return '!' <= c && c <= '~' + return isVchar(r) } -// quoteString renders a string as a RFC5322 quoted-string. +// quoteString renders a string as an RFC 5322 quoted-string. func quoteString(s string) string { var buf bytes.Buffer buf.WriteByte('"') - for _, c := range s { - ch := byte(c) - if isQtext(ch) || isWSP(ch) { - buf.WriteByte(ch) - } else if isVchar(ch) { + for _, r := range s { + if isQtext(r) || isWSP(r) { + buf.WriteRune(r) + } else if isVchar(r) { buf.WriteByte('\\') - buf.WriteByte(ch) + buf.WriteRune(r) } } buf.WriteByte('"') return buf.String() } -// isVchar reports whether c is an RFC 5322 VCHAR character. -func isVchar(c byte) bool { +// isVchar reports whether r is an RFC 5322 VCHAR character. +func isVchar(r rune) bool { // Visible (printing) characters. - return '!' <= c && c <= '~' + return '!' <= r && r <= '~' || isMultibyte(r) +} + +// isMultibyte reports whether r is a multi-byte UTF-8 character +// as supported by RFC 6532 +func isMultibyte(r rune) bool { + return r >= utf8.RuneSelf } -// isWSP reports whether c is a WSP (white space). -// WSP is a space or horizontal tab (RFC5234 Appendix B). -func isWSP(c byte) bool { - return c == ' ' || c == '\t' +// isWSP reports whether r is a WSP (white space). +// WSP is a space or horizontal tab (RFC 5234 Appendix B). +func isWSP(r rune) bool { + return r == ' ' || r == '\t' } diff --git a/libgo/go/net/mail/message_test.go b/libgo/go/net/mail/message_test.go index 4e718e2..bbbba6b 100644 --- a/libgo/go/net/mail/message_test.go +++ b/libgo/go/net/mail/message_test.go @@ -92,7 +92,7 @@ func TestDateParsing(t *testing.T) { "Fri, 21 Nov 1997 09:55:06 -0600", time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), }, - // RFC5322, Appendix A.6.2 + // RFC 5322, Appendix A.6.2 // Obsolete date. { "21 Nov 97 09:55:06 GMT", @@ -120,18 +120,24 @@ func TestDateParsing(t *testing.T) { } func TestAddressParsingError(t *testing.T) { - const txt = "=?iso-8859-2?Q?Bogl=E1rka_Tak=E1cs?= <unknown@gmail.com>" - _, err := ParseAddress(txt) - if err == nil || !strings.Contains(err.Error(), "charset not supported") { - t.Errorf(`mail.ParseAddress(%q) err: %q, want ".*charset not supported.*"`, txt, err) + mustErrTestCases := [...]struct { + text string + wantErrText string + }{ + 0: {"=?iso-8859-2?Q?Bogl=E1rka_Tak=E1cs?= <unknown@gmail.com>", "charset not supported"}, + 1: {"a@gmail.com b@gmail.com", "expected single address"}, + 2: {string([]byte{0xed, 0xa0, 0x80}) + " <micro@example.net>", "invalid utf-8 in address"}, + 3: {"\"" + string([]byte{0xed, 0xa0, 0x80}) + "\" <half-surrogate@example.com>", "invalid utf-8 in quoted-string"}, + 4: {"\"\\" + string([]byte{0x80}) + "\" <escaped-invalid-unicode@example.net>", "invalid utf-8 in quoted-string"}, + 5: {"\"\x00\" <null@example.net>", "bad character in quoted-string"}, + 6: {"\"\\\x00\" <escaped-null@example.net>", "bad character in quoted-string"}, } -} -func TestAddressParsingErrorUnquotedNonASCII(t *testing.T) { - const txt = "µ <micro@example.net>" - _, err := ParseAddress(txt) - if err == nil || !strings.Contains(err.Error(), "unencoded non-ASCII text in address") { - t.Errorf(`mail.ParseAddress(%q) err: %q, want ".*unencoded non-ASCII text in address.*"`, txt, err) + for i, tc := range mustErrTestCases { + _, err := ParseAddress(tc.text) + if err == nil || !strings.Contains(err.Error(), tc.wantErrText) { + t.Errorf(`mail.ParseAddress(%q) #%d want %q, got %v`, tc.text, i, tc.wantErrText, err) + } } } @@ -264,6 +270,46 @@ func TestAddressParsing(t *testing.T) { }, }, }, + // RFC 6532 3.2.3, qtext /= UTF8-non-ascii + { + `"Gø Pher" <gopher@example.com>`, + []*Address{ + { + Name: `Gø Pher`, + Address: "gopher@example.com", + }, + }, + }, + // RFC 6532 3.2, atext /= UTF8-non-ascii + { + `µ <micro@example.com>`, + []*Address{ + { + Name: `µ`, + Address: "micro@example.com", + }, + }, + }, + // RFC 6532 3.2.2, local address parts allow UTF-8 + { + `Micro <µ@example.com>`, + []*Address{ + { + Name: `Micro`, + Address: "µ@example.com", + }, + }, + }, + // RFC 6532 3.2.4, domains parts allow UTF-8 + { + `Micro <micro@µ.example.com>`, + []*Address{ + { + Name: `Micro`, + Address: "micro@µ.example.com", + }, + }, + }, } for _, test := range tests { if len(test.exp) == 1 { @@ -515,6 +561,11 @@ func TestAddressString(t *testing.T) { &Address{Name: "world?=", Address: "hello@world.com"}, `"world?=" <hello@world.com>`, }, + { + // should q-encode even for invalid utf-8. + &Address{Name: string([]byte{0xed, 0xa0, 0x80}), Address: "invalid-utf8@example.net"}, + "=?utf-8?q?=ED=A0=80?= <invalid-utf8@example.net>", + }, } for _, test := range tests { s := test.addr.String() @@ -610,7 +661,6 @@ func TestAddressParsingAndFormatting(t *testing.T) { `< @example.com>`, `<""test""blah""@example.com>`, `<""@0>`, - "<\"\t0\"@0>", } for _, test := range badTests { diff --git a/libgo/go/net/main_conf_test.go b/libgo/go/net/main_conf_test.go new file mode 100644 index 0000000..9875cea --- /dev/null +++ b/libgo/go/net/main_conf_test.go @@ -0,0 +1,38 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !nacl,!plan9,!windows + +package net + +// forceGoDNS forces the resolver configuration to use the pure Go resolver +// and returns a fixup function to restore the old settings. +func forceGoDNS() func() { + c := systemConf() + oldGo := c.netGo + oldCgo := c.netCgo + fixup := func() { + c.netGo = oldGo + c.netCgo = oldCgo + } + c.netGo = true + c.netCgo = false + return fixup +} + +// forceCgoDNS forces the resolver configuration to use the cgo resolver +// and returns a fixup function to restore the old settings. +// (On non-Unix systems forceCgoDNS returns nil.) +func forceCgoDNS() func() { + c := systemConf() + oldGo := c.netGo + oldCgo := c.netCgo + fixup := func() { + c.netGo = oldGo + c.netCgo = oldCgo + } + c.netGo = false + c.netCgo = true + return fixup +} diff --git a/libgo/go/net/non_unix_test.go b/libgo/go/net/main_noconf_test.go index db3427e..489477b 100644 --- a/libgo/go/net/non_unix_test.go +++ b/libgo/go/net/main_noconf_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -8,7 +8,7 @@ package net import "runtime" -// See unix_test.go for what these (don't) do. +// See main_conf_test.go for what these (don't) do. func forceGoDNS() func() { switch runtime.GOOS { case "plan9", "windows": @@ -18,5 +18,5 @@ func forceGoDNS() func() { } } -// See unix_test.go for what these (don't) do. +// See main_conf_test.go for what these (don't) do. func forceCgoDNS() func() { return nil } diff --git a/libgo/go/net/main_plan9_test.go b/libgo/go/net/main_plan9_test.go index 94501ca..2bc5be8 100644 --- a/libgo/go/net/main_plan9_test.go +++ b/libgo/go/net/main_plan9_test.go @@ -8,6 +8,7 @@ func installTestHooks() {} func uninstallTestHooks() {} +// forceCloseSockets must be called only from TestMain. func forceCloseSockets() {} func enableSocketConnect() {} diff --git a/libgo/go/net/main_test.go b/libgo/go/net/main_test.go index f3f8b1a..7573ded 100644 --- a/libgo/go/net/main_test.go +++ b/libgo/go/net/main_test.go @@ -26,8 +26,6 @@ var ( var ( testDNSFlood = flag.Bool("dnsflood", false, "whether to test DNS query flooding") - testExternal = flag.Bool("external", true, "allow use of external networks during long test") - // If external IPv4 connectivity exists, we can try dialing // non-node/interface local scope IPv4 addresses. // On Windows, Lookup APIs may not return IPv4-related diff --git a/libgo/go/net/main_unix_test.go b/libgo/go/net/main_unix_test.go index bfb4cd0..0cc129f 100644 --- a/libgo/go/net/main_unix_test.go +++ b/libgo/go/net/main_unix_test.go @@ -45,6 +45,7 @@ func uninstallTestHooks() { } } +// forceCloseSockets must be called only from TestMain. func forceCloseSockets() { for s := range sw.Sockets() { closeFunc(s) diff --git a/libgo/go/net/main_windows_test.go b/libgo/go/net/main_windows_test.go index 2d82974..6ea318c 100644 --- a/libgo/go/net/main_windows_test.go +++ b/libgo/go/net/main_windows_test.go @@ -11,6 +11,7 @@ var ( origConnect = connectFunc origConnectEx = connectExFunc origListen = listenFunc + origAccept = acceptFunc ) func installTestHooks() { @@ -19,6 +20,7 @@ func installTestHooks() { connectFunc = sw.Connect connectExFunc = sw.ConnectEx listenFunc = sw.Listen + acceptFunc = sw.AcceptEx } func uninstallTestHooks() { @@ -27,8 +29,10 @@ func uninstallTestHooks() { connectFunc = origConnect connectExFunc = origConnectEx listenFunc = origListen + acceptFunc = origAccept } +// forceCloseSockets must be called only from TestMain. func forceCloseSockets() { for s := range sw.Sockets() { closeFunc(s) diff --git a/libgo/go/net/mockserver_test.go b/libgo/go/net/mockserver_test.go index dd6f4df..766de6a 100644 --- a/libgo/go/net/mockserver_test.go +++ b/libgo/go/net/mockserver_test.go @@ -30,10 +30,20 @@ func testUnixAddr() string { func newLocalListener(network string) (Listener, error) { switch network { - case "tcp", "tcp4", "tcp6": + case "tcp": + if supportsIPv4 { + if ln, err := Listen("tcp4", "127.0.0.1:0"); err == nil { + return ln, nil + } + } + if supportsIPv6 { + return Listen("tcp6", "[::1]:0") + } + case "tcp4": if supportsIPv4 { return Listen("tcp4", "127.0.0.1:0") } + case "tcp6": if supportsIPv6 { return Listen("tcp6", "[::1]:0") } @@ -142,13 +152,6 @@ func (dss *dualStackServer) buildup(handler func(*dualStackServer, Listener)) er return nil } -func (dss *dualStackServer) putConn(c Conn) error { - dss.cmu.Lock() - dss.cs = append(dss.cs, c) - dss.cmu.Unlock() - return nil -} - func (dss *dualStackServer) teardownNetwork(network string) error { dss.lnmu.Lock() for i := range dss.lns { @@ -181,28 +184,24 @@ func (dss *dualStackServer) teardown() error { return nil } -func newDualStackServer(lns []streamListener) (*dualStackServer, error) { - dss := &dualStackServer{lns: lns, port: "0"} - for i := range dss.lns { - ln, err := Listen(dss.lns[i].network, JoinHostPort(dss.lns[i].address, dss.port)) - if err != nil { - for _, ln := range dss.lns[:i] { - ln.Listener.Close() - } - return nil, err - } - dss.lns[i].Listener = ln - dss.lns[i].done = make(chan bool) - if dss.port == "0" { - if _, dss.port, err = SplitHostPort(ln.Addr().String()); err != nil { - for _, ln := range dss.lns { - ln.Listener.Close() - } - return nil, err - } - } +func newDualStackServer() (*dualStackServer, error) { + lns, err := newDualStackListener() + if err != nil { + return nil, err + } + _, port, err := SplitHostPort(lns[0].Addr().String()) + if err != nil { + lns[0].Close() + lns[1].Close() + return nil, err } - return dss, nil + return &dualStackServer{ + lns: []streamListener{ + {network: "tcp4", address: lns[0].Addr().String(), Listener: lns[0], done: make(chan bool)}, + {network: "tcp6", address: lns[1].Addr().String(), Listener: lns[1], done: make(chan bool)}, + }, + port: port, + }, nil } func transponder(ln Listener, ch chan<- error) { @@ -225,7 +224,7 @@ func transponder(ln Listener, ch chan<- error) { defer c.Close() network := ln.Addr().Network() - if c.LocalAddr().Network() != network || c.LocalAddr().Network() != network { + if c.LocalAddr().Network() != network || c.RemoteAddr().Network() != network { ch <- fmt.Errorf("got %v->%v; expected %v->%v", c.LocalAddr().Network(), c.RemoteAddr().Network(), network, network) return } @@ -333,10 +332,18 @@ func timeoutTransmitter(c Conn, d, min, max time.Duration, ch chan<- error) { func newLocalPacketListener(network string) (PacketConn, error) { switch network { - case "udp", "udp4", "udp6": + case "udp": + if supportsIPv4 { + return ListenPacket("udp4", "127.0.0.1:0") + } + if supportsIPv6 { + return ListenPacket("udp6", "[::1]:0") + } + case "udp4": if supportsIPv4 { return ListenPacket("udp4", "127.0.0.1:0") } + case "udp6": if supportsIPv6 { return ListenPacket("udp6", "[::1]:0") } diff --git a/libgo/go/net/net.go b/libgo/go/net/net.go index d9d23fa..d6812d1 100644 --- a/libgo/go/net/net.go +++ b/libgo/go/net/net.go @@ -14,7 +14,7 @@ the same interfaces and similar Dial and Listen functions. The Dial function connects to a server: - conn, err := net.Dial("tcp", "google.com:80") + conn, err := net.Dial("tcp", "golang.org:80") if err != nil { // handle error } @@ -79,6 +79,7 @@ On Windows, the resolver always uses C library functions, such as GetAddrInfo an package net import ( + "context" "errors" "io" "os" @@ -297,7 +298,7 @@ func (c *conn) File() (f *os.File, err error) { // Multiple goroutines may invoke methods on a PacketConn simultaneously. type PacketConn interface { // ReadFrom reads a packet from the connection, - // copying the payload into b. It returns the number of + // copying the payload into b. It returns the number of // bytes copied into b and the return address that // was on the packet. // ReadFrom can be made to time out and return @@ -364,6 +365,9 @@ type Error interface { // Various errors contained in OpError. var ( + // For connection setup operations. + errNoSuitableAddress = errors.New("no suitable address found") + // For connection setup and write operations. errMissingAddress = errors.New("missing address") @@ -374,6 +378,22 @@ var ( ErrWriteToConnected = errors.New("use of WriteTo with pre-connected connection") ) +// mapErr maps from the context errors to the historical internal net +// error values. +// +// TODO(bradfitz): get rid of this after adjusting tests and making +// context.DeadlineExceeded implement net.Error? +func mapErr(err error) error { + switch err { + case context.Canceled: + return errCanceled + case context.DeadlineExceeded: + return errTimeout + default: + return err + } +} + // OpError is the error type usually returned by functions in the net // package. It describes the operation, network type, and address of // an error. diff --git a/libgo/go/net/net_test.go b/libgo/go/net/net_test.go index cd62b43..b2f825d 100644 --- a/libgo/go/net/net_test.go +++ b/libgo/go/net/net_test.go @@ -6,6 +6,7 @@ package net import ( "io" + "net/internal/socktest" "os" "runtime" "testing" @@ -304,3 +305,112 @@ func TestListenCloseListen(t *testing.T) { } t.Fatalf("failed to listen/close/listen on same address after %d tries", maxTries) } + +// See golang.org/issue/6163, golang.org/issue/6987. +func TestAcceptIgnoreAbortedConnRequest(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("%s does not have full support of socktest", runtime.GOOS) + } + + syserr := make(chan error) + go func() { + defer close(syserr) + for _, err := range abortedConnRequestErrors { + syserr <- err + } + }() + sw.Set(socktest.FilterAccept, func(so *socktest.Status) (socktest.AfterFilter, error) { + if err, ok := <-syserr; ok { + return nil, err + } + return nil, nil + }) + defer sw.Set(socktest.FilterAccept, nil) + + operr := make(chan error, 1) + handler := func(ls *localServer, ln Listener) { + defer close(operr) + c, err := ln.Accept() + if err != nil { + if perr := parseAcceptError(err); perr != nil { + operr <- perr + } + operr <- err + return + } + c.Close() + } + ls, err := newLocalServer("tcp") + if err != nil { + t.Fatal(err) + } + defer ls.teardown() + if err := ls.buildup(handler); err != nil { + t.Fatal(err) + } + + c, err := Dial(ls.Listener.Addr().Network(), ls.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + c.Close() + + for err := range operr { + t.Error(err) + } +} + +func TestZeroByteRead(t *testing.T) { + for _, network := range []string{"tcp", "unix", "unixpacket"} { + if !testableNetwork(network) { + t.Logf("skipping %s test", network) + continue + } + + ln, err := newLocalListener(network) + if err != nil { + t.Fatal(err) + } + connc := make(chan Conn, 1) + go func() { + defer ln.Close() + c, err := ln.Accept() + if err != nil { + t.Error(err) + } + connc <- c // might be nil + }() + c, err := Dial(network, ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer c.Close() + sc := <-connc + if sc == nil { + continue + } + defer sc.Close() + + if runtime.GOOS == "windows" { + // A zero byte read on Windows caused a wait for readability first. + // Rather than change that behavior, satisfy it in this test. + // See Issue 15735. + go io.WriteString(sc, "a") + } + + n, err := c.Read(nil) + if n != 0 || err != nil { + t.Errorf("%s: zero byte client read = %v, %v; want 0, nil", network, n, err) + } + + if runtime.GOOS == "windows" { + // Same as comment above. + go io.WriteString(c, "a") + } + n, err = sc.Read(nil) + if n != 0 || err != nil { + t.Errorf("%s: zero byte server read = %v, %v; want 0, nil", network, n, err) + } + } +} diff --git a/libgo/go/net/packetconn_test.go b/libgo/go/net/packetconn_test.go index 7f3ea8a..7d50489 100644 --- a/libgo/go/net/packetconn_test.go +++ b/libgo/go/net/packetconn_test.go @@ -1,4 +1,4 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/parse.go b/libgo/go/net/parse.go index 80836a1..2c6b98a 100644 --- a/libgo/go/net/parse.go +++ b/libgo/go/net/parse.go @@ -106,14 +106,14 @@ func splitAtBytes(s string, t string) []string { for i := 0; i < len(s); i++ { if byteIndex(t, s[i]) >= 0 { if last < i { - a[n] = string(s[last:i]) + a[n] = s[last:i] n++ } last = i + 1 } } if last < len(s) { - a[n] = string(s[last:]) + a[n] = s[last:] n++ } return a[0:n] diff --git a/libgo/go/net/pipe.go b/libgo/go/net/pipe.go index 5fc830b..37e552f 100644 --- a/libgo/go/net/pipe.go +++ b/libgo/go/net/pipe.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/pipe_test.go b/libgo/go/net/pipe_test.go index 60c3920..e3172d8 100644 --- a/libgo/go/net/pipe_test.go +++ b/libgo/go/net/pipe_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/platform_test.go b/libgo/go/net/platform_test.go index 76c5313..2a14095 100644 --- a/libgo/go/net/platform_test.go +++ b/libgo/go/net/platform_test.go @@ -5,6 +5,7 @@ package net import ( + "internal/testenv" "os" "runtime" "strings" @@ -110,7 +111,7 @@ func testableListenArgs(network, address, client string) bool { } // Test wildcard IP addresses. - if wildcard && (testing.Short() || !*testExternal) { + if wildcard && !testenv.HasExternalNetwork() { return false } diff --git a/libgo/go/net/port.go b/libgo/go/net/port.go new file mode 100644 index 0000000..8e1321a --- /dev/null +++ b/libgo/go/net/port.go @@ -0,0 +1,62 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +// parsePort parses service as a decimal interger and returns the +// corresponding value as port. It is the caller's responsibility to +// parse service as a non-decimal integer when needsLookup is true. +// +// Some system resolvers will return a valid port number when given a number +// over 65536 (see https://github.com/golang/go/issues/11715). Alas, the parser +// can't bail early on numbers > 65536. Therefore reasonably large/small +// numbers are parsed in full and rejected if invalid. +func parsePort(service string) (port int, needsLookup bool) { + if service == "" { + // Lock in the legacy behavior that an empty string + // means port 0. See golang.org/issue/13610. + return 0, false + } + const ( + max = uint32(1<<32 - 1) + cutoff = uint32(1 << 30) + ) + neg := false + if service[0] == '+' { + service = service[1:] + } else if service[0] == '-' { + neg = true + service = service[1:] + } + var n uint32 + for _, d := range service { + if '0' <= d && d <= '9' { + d -= '0' + } else { + return 0, true + } + if n >= cutoff { + n = max + break + } + n *= 10 + nn := n + uint32(d) + if nn < n || nn > max { + n = max + break + } + n = nn + } + if !neg && n >= cutoff { + port = int(cutoff - 1) + } else if neg && n > cutoff { + port = int(cutoff) + } else { + port = int(n) + } + if neg { + port = -port + } + return port, false +} diff --git a/libgo/go/net/port_test.go b/libgo/go/net/port_test.go new file mode 100644 index 0000000..e0bdb42 --- /dev/null +++ b/libgo/go/net/port_test.go @@ -0,0 +1,52 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "testing" + +var parsePortTests = []struct { + service string + port int + needsLookup bool +}{ + {"", 0, false}, + + // Decimal number literals + {"-1073741825", -1 << 30, false}, + {"-1073741824", -1 << 30, false}, + {"-1073741823", -(1<<30 - 1), false}, + {"-123456789", -123456789, false}, + {"-1", -1, false}, + {"-0", 0, false}, + {"0", 0, false}, + {"+0", 0, false}, + {"+1", 1, false}, + {"65535", 65535, false}, + {"65536", 65536, false}, + {"123456789", 123456789, false}, + {"1073741822", 1<<30 - 2, false}, + {"1073741823", 1<<30 - 1, false}, + {"1073741824", 1<<30 - 1, false}, + {"1073741825", 1<<30 - 1, false}, + + // Others + {"abc", 0, true}, + {"9pfs", 0, true}, + {"123badport", 0, true}, + {"bad123port", 0, true}, + {"badport123", 0, true}, + {"123456789badport", 0, true}, + {"-2147483649badport", 0, true}, + {"2147483649badport", 0, true}, +} + +func TestParsePort(t *testing.T) { + // The following test cases are cribbed from the strconv + for _, tt := range parsePortTests { + if port, needsLookup := parsePort(tt.service); port != tt.port || needsLookup != tt.needsLookup { + t.Errorf("parsePort(%q) = %d, %t; want %d, %t", tt.service, port, needsLookup, tt.port, tt.needsLookup) + } + } +} diff --git a/libgo/go/net/protoconn_test.go b/libgo/go/net/protoconn_test.go index c6ef23b..23589d3 100644 --- a/libgo/go/net/protoconn_test.go +++ b/libgo/go/net/protoconn_test.go @@ -1,4 +1,4 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/rpc/client.go b/libgo/go/net/rpc/client.go index d0c4a69..862fb1a 100644 --- a/libgo/go/net/rpc/client.go +++ b/libgo/go/net/rpc/client.go @@ -55,7 +55,7 @@ type Client struct { // reading of RPC responses for the client side of an RPC session. // The client calls WriteRequest to write a request to the connection // and calls ReadResponseHeader and ReadResponseBody in pairs -// to read responses. The client calls Close when finished with the +// to read responses. The client calls Close when finished with the // connection. ReadResponseBody may be called with a nil // argument to force the body of the response to be read and then // discarded. @@ -173,7 +173,7 @@ func (call *Call) done() { case call.Done <- call: // ok default: - // We don't want to block here. It is the caller's responsibility to make + // We don't want to block here. It is the caller's responsibility to make // sure the channel has enough buffer space. See comment in Go(). if debugLog { log.Println("rpc: discarding Call reply due to insufficient Done chan capacity") @@ -285,9 +285,9 @@ func (client *Client) Close() error { return client.codec.Close() } -// Go invokes the function asynchronously. It returns the Call structure representing -// the invocation. The done channel will signal when the call is complete by returning -// the same Call object. If done is nil, Go will allocate a new channel. +// Go invokes the function asynchronously. It returns the Call structure representing +// the invocation. The done channel will signal when the call is complete by returning +// the same Call object. If done is nil, Go will allocate a new channel. // If non-nil, done must be buffered or Go will deliberately crash. func (client *Client) Go(serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call { call := new(Call) @@ -299,7 +299,7 @@ func (client *Client) Go(serviceMethod string, args interface{}, reply interface } else { // If caller passes done != nil, it must arrange that // done has enough buffer for the number of simultaneous - // RPCs that will be using that channel. If the channel + // RPCs that will be using that channel. If the channel // is totally unbuffered, it's best not to run at all. if cap(done) == 0 { log.Panic("rpc: done channel is unbuffered") diff --git a/libgo/go/net/rpc/jsonrpc/all_test.go b/libgo/go/net/rpc/jsonrpc/all_test.go index a433a36..b811d3c 100644 --- a/libgo/go/net/rpc/jsonrpc/all_test.go +++ b/libgo/go/net/rpc/jsonrpc/all_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/rpc/jsonrpc/client.go b/libgo/go/net/rpc/jsonrpc/client.go index 2194f21..da1b816 100644 --- a/libgo/go/net/rpc/jsonrpc/client.go +++ b/libgo/go/net/rpc/jsonrpc/client.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/rpc/jsonrpc/server.go b/libgo/go/net/rpc/jsonrpc/server.go index e6d37cf..40e4e6f 100644 --- a/libgo/go/net/rpc/jsonrpc/server.go +++ b/libgo/go/net/rpc/jsonrpc/server.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -110,7 +110,7 @@ func (c *serverCodec) WriteResponse(r *rpc.Response, x interface{}) error { c.mutex.Unlock() if b == nil { - // Invalid request so no id. Use JSON null. + // Invalid request so no id. Use JSON null. b = &null } resp := serverResponse{Id: b} diff --git a/libgo/go/net/rpc/server.go b/libgo/go/net/rpc/server.go index c4d4479..cff3241 100644 --- a/libgo/go/net/rpc/server.go +++ b/libgo/go/net/rpc/server.go @@ -143,8 +143,8 @@ const ( DefaultDebugPath = "/debug/rpc" ) -// Precompute the reflect type for error. Can't use error directly -// because Typeof takes an empty interface value. This is annoying. +// Precompute the reflect type for error. Can't use error directly +// because Typeof takes an empty interface value. This is annoying. var typeOfError = reflect.TypeOf((*error)(nil)).Elem() type methodType struct { @@ -162,7 +162,7 @@ type service struct { method map[string]*methodType // registered methods } -// Request is a header written before every RPC call. It is used internally +// Request is a header written before every RPC call. It is used internally // but documented here as an aid to debugging, such as when analyzing // network traffic. type Request struct { @@ -171,7 +171,7 @@ type Request struct { next *Request // for free list in Server } -// Response is a header written before every RPC return. It is used internally +// Response is a header written before every RPC return. It is used internally // but documented here as an aid to debugging, such as when analyzing // network traffic. type Response struct { @@ -442,7 +442,7 @@ func (c *gobServerCodec) Close() error { // ServeConn blocks, serving the connection until the client hangs up. // The caller typically invokes ServeConn in a go statement. // ServeConn uses the gob wire format (see package gob) on the -// connection. To use an alternate codec, use ServeCodec. +// connection. To use an alternate codec, use ServeCodec. func (server *Server) ServeConn(conn io.ReadWriteCloser) { buf := bufio.NewWriter(conn) srv := &gobServerCodec{ @@ -583,7 +583,7 @@ func (server *Server) readRequestHeader(codec ServerCodec) (service *service, mt return } - // We read the header successfully. If we see an error now, + // We read the header successfully. If we see an error now, // we can still recover and move on to the next request. keepReading = true @@ -638,7 +638,7 @@ func RegisterName(name string, rcvr interface{}) error { // RPC responses for the server side of an RPC session. // The server calls ReadRequestHeader and ReadRequestBody in pairs // to read requests from the connection, and it calls WriteResponse to -// write a response back. The server calls Close when finished with the +// write a response back. The server calls Close when finished with the // connection. ReadRequestBody may be called with a nil // argument to force the body of the request to be read and discarded. type ServerCodec interface { @@ -654,7 +654,7 @@ type ServerCodec interface { // ServeConn blocks, serving the connection until the client hangs up. // The caller typically invokes ServeConn in a go statement. // ServeConn uses the gob wire format (see package gob) on the -// connection. To use an alternate codec, use ServeCodec. +// connection. To use an alternate codec, use ServeCodec. func ServeConn(conn io.ReadWriteCloser) { DefaultServer.ServeConn(conn) } diff --git a/libgo/go/net/rpc/server_test.go b/libgo/go/net/rpc/server_test.go index 8871c88..d04271d 100644 --- a/libgo/go/net/rpc/server_test.go +++ b/libgo/go/net/rpc/server_test.go @@ -183,7 +183,7 @@ func testRPC(t *testing.T, addr string) { err = client.Call("Arith.Unknown", args, reply) if err == nil { t.Error("expected error calling unknown service") - } else if strings.Index(err.Error(), "method") < 0 { + } else if !strings.Contains(err.Error(), "method") { t.Error("expected error about method; got", err) } @@ -226,7 +226,7 @@ func testRPC(t *testing.T, addr string) { err = client.Call("Arith.Add", reply, reply) // args, reply would be the correct thing to use if err == nil { t.Error("expected error calling Arith.Add with wrong arg type") - } else if strings.Index(err.Error(), "type") < 0 { + } else if !strings.Contains(err.Error(), "type") { t.Error("expected error about type; got", err) } @@ -657,6 +657,9 @@ func benchmarkEndToEnd(dial func() (*Client, error), b *testing.B) { } func benchmarkEndToEndAsync(dial func() (*Client, error), b *testing.B) { + if b.N == 0 { + return + } const MaxConcurrentCalls = 100 once.Do(startServer) client, err := dial() diff --git a/libgo/go/net/sendfile_dragonfly.go b/libgo/go/net/sendfile_dragonfly.go index a9cf3fe..d4b825c 100644 --- a/libgo/go/net/sendfile_dragonfly.go +++ b/libgo/go/net/sendfile_dragonfly.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -53,7 +53,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { // use the current position of the file -- if you pass it offset 0, it starts // from offset 0. There's no way to tell it "start from current position", so // we have to manage that explicitly. - pos, err := f.Seek(0, os.SEEK_CUR) + pos, err := f.Seek(0, io.SeekCurrent) if err != nil { return 0, err, false } @@ -81,7 +81,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { break } if err1 == syscall.EAGAIN { - if err1 = c.pd.WaitWrite(); err1 == nil { + if err1 = c.pd.waitWrite(); err1 == nil { continue } } diff --git a/libgo/go/net/sendfile_freebsd.go b/libgo/go/net/sendfile_freebsd.go index d0bf603..18cbb27 100644 --- a/libgo/go/net/sendfile_freebsd.go +++ b/libgo/go/net/sendfile_freebsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -53,7 +53,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { // use the current position of the file -- if you pass it offset 0, it starts // from offset 0. There's no way to tell it "start from current position", so // we have to manage that explicitly. - pos, err := f.Seek(0, os.SEEK_CUR) + pos, err := f.Seek(0, io.SeekCurrent) if err != nil { return 0, err, false } @@ -81,7 +81,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { break } if err1 == syscall.EAGAIN { - if err1 = c.pd.WaitWrite(); err1 == nil { + if err1 = c.pd.waitWrite(); err1 == nil { continue } } diff --git a/libgo/go/net/sendfile_linux.go b/libgo/go/net/sendfile_linux.go index 5ca41c39e..7e741f9 100644 --- a/libgo/go/net/sendfile_linux.go +++ b/libgo/go/net/sendfile_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -57,7 +57,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { break } if err1 == syscall.EAGAIN { - if err1 = c.pd.WaitWrite(); err1 == nil { + if err1 = c.pd.waitWrite(); err1 == nil { continue } } diff --git a/libgo/go/net/sendfile_solaris.go b/libgo/go/net/sendfile_solaris.go index f683381..add70c3 100644 --- a/libgo/go/net/sendfile_solaris.go +++ b/libgo/go/net/sendfile_solaris.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -26,8 +26,6 @@ const maxSendfileSize int = 4 << 20 // // if handled == false, sendFile performed no work. func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { - return // Solaris sendfile is disabled until Issue 13892 is understood and fixed - // Solaris uses 0 as the "until EOF" value. If you pass in more bytes than the // file contains, it will loop back to the beginning ad nauseam until it's sent // exactly the number of bytes told to. As such, we need to know exactly how many @@ -59,7 +57,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { // use the current position of the file -- if you pass it offset 0, it starts // from offset 0. There's no way to tell it "start from current position", so // we have to manage that explicitly. - pos, err := f.Seek(0, os.SEEK_CUR) + pos, err := f.Seek(0, io.SeekCurrent) if err != nil { return 0, err, false } @@ -78,6 +76,13 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { } pos1 := pos n, err1 := syscall.Sendfile(dst, src, &pos1, n) + if err1 == syscall.EAGAIN || err1 == syscall.EINTR { + // partial write may have occurred + if n = int(pos1 - pos); n == 0 { + // nothing more to write + err1 = nil + } + } if n > 0 { pos += int64(n) written += int64(n) @@ -87,7 +92,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { break } if err1 == syscall.EAGAIN { - if err1 = c.pd.WaitWrite(); err1 == nil { + if err1 = c.pd.waitWrite(); err1 == nil { continue } } diff --git a/libgo/go/net/sendfile_stub.go b/libgo/go/net/sendfile_stub.go index a0760b4..905f1d6 100644 --- a/libgo/go/net/sendfile_stub.go +++ b/libgo/go/net/sendfile_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sendfile_test.go b/libgo/go/net/sendfile_test.go new file mode 100644 index 0000000..2255e7c --- /dev/null +++ b/libgo/go/net/sendfile_test.go @@ -0,0 +1,90 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "testing" +) + +const ( + twain = "testdata/Mark.Twain-Tom.Sawyer.txt" + twainLen = 387851 + twainSHA256 = "461eb7cb2d57d293fc680c836464c9125e4382be3596f7d415093ae9db8fcb0e" +) + +func TestSendfile(t *testing.T) { + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + + errc := make(chan error, 1) + go func(ln Listener) { + // Wait for a connection. + conn, err := ln.Accept() + if err != nil { + errc <- err + close(errc) + return + } + + go func() { + defer close(errc) + defer conn.Close() + + f, err := os.Open(twain) + if err != nil { + errc <- err + return + } + defer f.Close() + + // Return file data using io.Copy, which should use + // sendFile if available. + sbytes, err := io.Copy(conn, f) + if err != nil { + errc <- err + return + } + + if sbytes != twainLen { + errc <- fmt.Errorf("sent %d bytes; expected %d", sbytes, twainLen) + return + } + }() + }(ln) + + // Connect to listener to retrieve file and verify digest matches + // expected. + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + h := sha256.New() + rbytes, err := io.Copy(h, c) + if err != nil { + t.Error(err) + } + + if rbytes != twainLen { + t.Errorf("received %d bytes; expected %d", rbytes, twainLen) + } + + if res := hex.EncodeToString(h.Sum(nil)); res != twainSHA256 { + t.Error("retrieved data hash did not match") + } + + for err := range errc { + t.Error(err) + } +} diff --git a/libgo/go/net/sendfile_windows.go b/libgo/go/net/sendfile_windows.go index f3f3b54..bc0b7fb 100644 --- a/libgo/go/net/sendfile_windows.go +++ b/libgo/go/net/sendfile_windows.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -18,7 +18,7 @@ import ( // // if handled == false, sendFile performed no work. // -// Note that sendfile for windows does not suppport >2GB file. +// Note that sendfile for windows does not support >2GB file. func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) { var n int64 = 0 // by default, copy until EOF diff --git a/libgo/go/net/smtp/smtp.go b/libgo/go/net/smtp/smtp.go index 0988350..9e04dd7 100644 --- a/libgo/go/net/smtp/smtp.go +++ b/libgo/go/net/smtp/smtp.go @@ -8,6 +8,11 @@ // AUTH RFC 2554 // STARTTLS RFC 3207 // Additional extensions may be handled by clients. +// +// The smtp package is frozen and not accepting new features. +// Some external packages provide more functionality. See: +// +// https://godoc.org/?q=smtp package smtp import ( @@ -83,8 +88,8 @@ func (c *Client) hello() error { // Hello sends a HELO or EHLO to the server as the given host name. // Calling this method is only necessary if the client needs control -// over the host name used. The client will introduce itself as "localhost" -// automatically otherwise. If Hello is called, it must be called before +// over the host name used. The client will introduce itself as "localhost" +// automatically otherwise. If Hello is called, it must be called before // any of the other methods. func (c *Client) Hello(localName string) error { if c.didHello { @@ -265,7 +270,7 @@ func (d *dataCloser) Close() error { // Data issues a DATA command to the server and returns a writer that // can be used to write the mail headers and body. The caller should -// close the writer before calling any more methods on c. A call to +// close the writer before calling any more methods on c. A call to // Data must be preceded by one or more calls to Rcpt. func (c *Client) Data() (io.WriteCloser, error) { _, _, err := c.cmd(354, "DATA") @@ -287,7 +292,7 @@ var testHookStartTLS func(*tls.Config) // nil, except for tests // // The msg parameter should be an RFC 822-style email with headers // first, a blank line, and then the message body. The lines of msg -// should be CRLF terminated. The msg headers should usually include +// should be CRLF terminated. The msg headers should usually include // fields such as "From", "To", "Subject", and "Cc". Sending "Bcc" // messages is accomplished by including an email address in the to // parameter but not including it in the msg headers. diff --git a/libgo/go/net/sock_bsd.go b/libgo/go/net/sock_bsd.go index 6c37109..4e0e9e0 100644 --- a/libgo/go/net/sock_bsd.go +++ b/libgo/go/net/sock_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sock_linux.go b/libgo/go/net/sock_linux.go index cc5ce15..e2732c5 100644 --- a/libgo/go/net/sock_linux.go +++ b/libgo/go/net/sock_linux.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sock_plan9.go b/libgo/go/net/sock_plan9.go index 88d9ed1..9367ad8 100644 --- a/libgo/go/net/sock_plan9.go +++ b/libgo/go/net/sock_plan9.go @@ -1,4 +1,4 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sock_posix.go b/libgo/go/net/sock_posix.go index 4676721..c3af27b 100644 --- a/libgo/go/net/sock_posix.go +++ b/libgo/go/net/sock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,9 +7,9 @@ package net import ( + "context" "os" "syscall" - "time" ) // A sockaddr represents a TCP, UDP, IP or Unix network endpoint @@ -34,7 +34,7 @@ type sockaddr interface { // socket returns a network file descriptor that is ready for // asynchronous I/O using the network poller. -func socket(net string, family, sotype, proto int, ipv6only bool, laddr, raddr sockaddr, deadline time.Time, cancel <-chan struct{}) (fd *netFD, err error) { +func socket(ctx context.Context, net string, family, sotype, proto int, ipv6only bool, laddr, raddr sockaddr) (fd *netFD, err error) { s, err := sysSocket(family, sotype, proto) if err != nil { return nil, err @@ -86,7 +86,7 @@ func socket(net string, family, sotype, proto int, ipv6only bool, laddr, raddr s return fd, nil } } - if err := fd.dial(laddr, raddr, deadline, cancel); err != nil { + if err := fd.dial(ctx, laddr, raddr); err != nil { fd.Close() return nil, err } @@ -117,7 +117,7 @@ func (fd *netFD) addrFunc() func(syscall.Sockaddr) Addr { return func(syscall.Sockaddr) Addr { return nil } } -func (fd *netFD) dial(laddr, raddr sockaddr, deadline time.Time, cancel <-chan struct{}) error { +func (fd *netFD) dial(ctx context.Context, laddr, raddr sockaddr) error { var err error var lsa syscall.Sockaddr if laddr != nil { @@ -134,7 +134,7 @@ func (fd *netFD) dial(laddr, raddr sockaddr, deadline time.Time, cancel <-chan s if rsa, err = raddr.sockaddr(fd.family); err != nil { return err } - if err := fd.connect(lsa, rsa, deadline, cancel); err != nil { + if err := fd.connect(ctx, lsa, rsa); err != nil { return err } fd.isConnected = true diff --git a/libgo/go/net/sock_stub.go b/libgo/go/net/sock_stub.go index ed6b089..5ac1e86 100644 --- a/libgo/go/net/sock_stub.go +++ b/libgo/go/net/sock_stub.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sock_windows.go b/libgo/go/net/sock_windows.go index 888e70bb..89a3ca4 100644 --- a/libgo/go/net/sock_windows.go +++ b/libgo/go/net/sock_windows.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockopt_bsd.go b/libgo/go/net/sockopt_bsd.go index 52b2a5d..734a109 100644 --- a/libgo/go/net/sockopt_bsd.go +++ b/libgo/go/net/sockopt_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -27,7 +27,7 @@ func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { } if supportsIPv4map && family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { // Allow both IP versions even if the OS default - // is otherwise. Note that some operating systems + // is otherwise. Note that some operating systems // never admit this option. syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) } diff --git a/libgo/go/net/sockopt_linux.go b/libgo/go/net/sockopt_linux.go index 54c20b1..0f70b12 100644 --- a/libgo/go/net/sockopt_linux.go +++ b/libgo/go/net/sockopt_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -12,7 +12,7 @@ import ( func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { // Allow both IP versions even if the OS default - // is otherwise. Note that some operating systems + // is otherwise. Note that some operating systems // never admit this option. syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) } diff --git a/libgo/go/net/sockopt_plan9.go b/libgo/go/net/sockopt_plan9.go index 8bc689b..02468cd 100644 --- a/libgo/go/net/sockopt_plan9.go +++ b/libgo/go/net/sockopt_plan9.go @@ -1,9 +1,11 @@ -// Copyright 2014 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import "syscall" + func setKeepAlive(fd *netFD, keepalive bool) error { if keepalive { _, e := fd.ctl.WriteAt([]byte("keepalive"), 0) @@ -11,3 +13,7 @@ func setKeepAlive(fd *netFD, keepalive bool) error { } return nil } + +func setLinger(fd *netFD, sec int) error { + return syscall.EPLAN9 +} diff --git a/libgo/go/net/sockopt_posix.go b/libgo/go/net/sockopt_posix.go index 1654d1b..cd3d562 100644 --- a/libgo/go/net/sockopt_posix.go +++ b/libgo/go/net/sockopt_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockopt_solaris.go b/libgo/go/net/sockopt_solaris.go index 54c20b1..0f70b12 100644 --- a/libgo/go/net/sockopt_solaris.go +++ b/libgo/go/net/sockopt_solaris.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -12,7 +12,7 @@ import ( func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { // Allow both IP versions even if the OS default - // is otherwise. Note that some operating systems + // is otherwise. Note that some operating systems // never admit this option. syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) } diff --git a/libgo/go/net/sockopt_stub.go b/libgo/go/net/sockopt_stub.go index de5ee0b..7e9e560 100644 --- a/libgo/go/net/sockopt_stub.go +++ b/libgo/go/net/sockopt_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockopt_windows.go b/libgo/go/net/sockopt_windows.go index cb64a40..8017426 100644 --- a/libgo/go/net/sockopt_windows.go +++ b/libgo/go/net/sockopt_windows.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -12,7 +12,7 @@ import ( func setDefaultSockopts(s syscall.Handle, family, sotype int, ipv6only bool) error { if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { // Allow both IP versions even if the OS default - // is otherwise. Note that some operating systems + // is otherwise. Note that some operating systems // never admit this option. syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) } diff --git a/libgo/go/net/sockoptip_bsd.go b/libgo/go/net/sockoptip_bsd.go index 2199e48..b15c639 100644 --- a/libgo/go/net/sockoptip_bsd.go +++ b/libgo/go/net/sockoptip_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockoptip_linux.go b/libgo/go/net/sockoptip_linux.go index a69b778..c1dcc91 100644 --- a/libgo/go/net/sockoptip_linux.go +++ b/libgo/go/net/sockoptip_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockoptip_posix.go b/libgo/go/net/sockoptip_posix.go index c2579be..d508860 100644 --- a/libgo/go/net/sockoptip_posix.go +++ b/libgo/go/net/sockoptip_posix.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockoptip_stub.go b/libgo/go/net/sockoptip_stub.go index 32ec5dd..f698687 100644 --- a/libgo/go/net/sockoptip_stub.go +++ b/libgo/go/net/sockoptip_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockoptip_windows.go b/libgo/go/net/sockoptip_windows.go index 7b11f20..916debe 100644 --- a/libgo/go/net/sockoptip_windows.go +++ b/libgo/go/net/sockoptip_windows.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsock.go b/libgo/go/net/tcpsock.go index 8765aff..7cffcc5 100644 --- a/libgo/go/net/tcpsock.go +++ b/libgo/go/net/tcpsock.go @@ -1,9 +1,17 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import ( + "context" + "io" + "os" + "syscall" + "time" +) + // TCPAddr represents the address of a TCP end point. type TCPAddr struct { IP IP @@ -53,9 +61,234 @@ func ResolveTCPAddr(net, addr string) (*TCPAddr, error) { default: return nil, UnknownNetworkError(net) } - addrs, err := internetAddrList(net, addr, noDeadline) + addrs, err := internetAddrList(context.Background(), net, addr) if err != nil { return nil, err } return addrs.first(isIPv4).(*TCPAddr), nil } + +// TCPConn is an implementation of the Conn interface for TCP network +// connections. +type TCPConn struct { + conn +} + +// ReadFrom implements the io.ReaderFrom ReadFrom method. +func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.readFrom(r) + if err != nil && err != io.EOF { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, err +} + +// CloseRead shuts down the reading side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseRead() error { + if !c.ok() { + return syscall.EINVAL + } + if err := c.fd.closeRead(); err != nil { + return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// CloseWrite shuts down the writing side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseWrite() error { + if !c.ok() { + return syscall.EINVAL + } + if err := c.fd.closeWrite(); err != nil { + return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// SetLinger sets the behavior of Close on a connection which still +// has data waiting to be sent or to be acknowledged. +// +// If sec < 0 (the default), the operating system finishes sending the +// data in the background. +// +// If sec == 0, the operating system discards any unsent or +// unacknowledged data. +// +// If sec > 0, the data is sent in the background as with sec < 0. On +// some operating systems after sec seconds have elapsed any remaining +// unsent data may be discarded. +func (c *TCPConn) SetLinger(sec int) error { + if !c.ok() { + return syscall.EINVAL + } + if err := setLinger(c.fd, sec); err != nil { + return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// SetKeepAlive sets whether the operating system should send +// keepalive messages on the connection. +func (c *TCPConn) SetKeepAlive(keepalive bool) error { + if !c.ok() { + return syscall.EINVAL + } + if err := setKeepAlive(c.fd, keepalive); err != nil { + return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// SetKeepAlivePeriod sets period between keep alives. +func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { + if !c.ok() { + return syscall.EINVAL + } + if err := setKeepAlivePeriod(c.fd, d); err != nil { + return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// SetNoDelay controls whether the operating system should delay +// packet transmission in hopes of sending fewer packets (Nagle's +// algorithm). The default is true (no delay), meaning that data is +// sent as soon as possible after a Write. +func (c *TCPConn) SetNoDelay(noDelay bool) error { + if !c.ok() { + return syscall.EINVAL + } + if err := setNoDelay(c.fd, noDelay); err != nil { + return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +func newTCPConn(fd *netFD) *TCPConn { + c := &TCPConn{conn{fd}} + setNoDelay(c.fd, true) + return c +} + +// DialTCP connects to the remote address raddr on the network net, +// which must be "tcp", "tcp4", or "tcp6". If laddr is not nil, it is +// used as the local address for the connection. +func DialTCP(net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + switch net { + case "tcp", "tcp4", "tcp6": + default: + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + } + if raddr == nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + } + c, err := dialTCP(context.Background(), net, laddr, raddr) + if err != nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + } + return c, nil +} + +// TCPListener is a TCP network listener. Clients should typically +// use variables of type Listener instead of assuming TCP. +type TCPListener struct { + fd *netFD +} + +// AcceptTCP accepts the next incoming call and returns the new +// connection. +func (l *TCPListener) AcceptTCP() (*TCPConn, error) { + if !l.ok() { + return nil, syscall.EINVAL + } + c, err := l.accept() + if err != nil { + return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return c, nil +} + +// Accept implements the Accept method in the Listener interface; it +// waits for the next call and returns a generic Conn. +func (l *TCPListener) Accept() (Conn, error) { + if !l.ok() { + return nil, syscall.EINVAL + } + c, err := l.accept() + if err != nil { + return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return c, nil +} + +// Close stops listening on the TCP address. +// Already Accepted connections are not closed. +func (l *TCPListener) Close() error { + if !l.ok() { + return syscall.EINVAL + } + if err := l.close(); err != nil { + return &OpError{Op: "close", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return nil +} + +// Addr returns the listener's network address, a *TCPAddr. +// The Addr returned is shared by all invocations of Addr, so +// do not modify it. +func (l *TCPListener) Addr() Addr { return l.fd.laddr } + +// SetDeadline sets the deadline associated with the listener. +// A zero time value disables the deadline. +func (l *TCPListener) SetDeadline(t time.Time) error { + if !l.ok() { + return syscall.EINVAL + } + if err := l.fd.setDeadline(t); err != nil { + return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return nil +} + +// File returns a copy of the underlying os.File, set to blocking +// mode. It is the caller's responsibility to close f when finished. +// Closing l does not affect f, and closing f does not affect l. +// +// The returned os.File's file descriptor is different from the +// connection's. Attempting to change properties of the original +// using this duplicate may or may not have the desired effect. +func (l *TCPListener) File() (f *os.File, err error) { + if !l.ok() { + return nil, syscall.EINVAL + } + f, err = l.file() + if err != nil { + return nil, &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return +} + +// ListenTCP announces on the TCP address laddr and returns a TCP +// listener. Net must be "tcp", "tcp4", or "tcp6". If laddr has a +// port of 0, ListenTCP will choose an available port. The caller can +// use the Addr method of TCPListener to retrieve the chosen address. +func ListenTCP(net string, laddr *TCPAddr) (*TCPListener, error) { + switch net { + case "tcp", "tcp4", "tcp6": + default: + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + } + if laddr == nil { + laddr = &TCPAddr{} + } + ln, err := listenTCP(context.Background(), net, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return ln, nil +} diff --git a/libgo/go/net/tcpsock_plan9.go b/libgo/go/net/tcpsock_plan9.go index afccbfe..d286060 100644 --- a/libgo/go/net/tcpsock_plan9.go +++ b/libgo/go/net/tcpsock_plan9.go @@ -1,230 +1,73 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "io" "os" - "syscall" - "time" ) -// TCPConn is an implementation of the Conn interface for TCP network -// connections. -type TCPConn struct { - conn +func (c *TCPConn) readFrom(r io.Reader) (int64, error) { + return genericReadFrom(c, r) } -func newTCPConn(fd *netFD) *TCPConn { - return &TCPConn{conn{fd}} -} - -// ReadFrom implements the io.ReaderFrom ReadFrom method. -func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { - n, err := genericReadFrom(c, r) - if err != nil && err != io.EOF { - err = &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return n, err -} - -// CloseRead shuts down the reading side of the TCP connection. -// Most callers should just use Close. -func (c *TCPConn) CloseRead() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeRead() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} +func dialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + if testHookDialTCP != nil { + return testHookDialTCP(ctx, net, laddr, raddr) } - return err -} - -// CloseWrite shuts down the writing side of the TCP connection. -// Most callers should just use Close. -func (c *TCPConn) CloseWrite() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeWrite() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err -} - -// SetLinger sets the behavior of Close on a connection which still -// has data waiting to be sent or to be acknowledged. -// -// If sec < 0 (the default), the operating system finishes sending the -// data in the background. -// -// If sec == 0, the operating system discards any unsent or -// unacknowledged data. -// -// If sec > 0, the data is sent in the background as with sec < 0. On -// some operating systems after sec seconds have elapsed any remaining -// unsent data may be discarded. -func (c *TCPConn) SetLinger(sec int) error { - return &OpError{Op: "set", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} -} - -// SetKeepAlive sets whether the operating system should send -// keepalive messages on the connection. -func (c *TCPConn) SetKeepAlive(keepalive bool) error { - if !c.ok() { - return syscall.EPLAN9 - } - if err := setKeepAlive(c.fd, keepalive); err != nil { - return &OpError{Op: "set", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// SetKeepAlivePeriod sets period between keep alives. -func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { - if !c.ok() { - return syscall.EPLAN9 - } - if err := setKeepAlivePeriod(c.fd, d); err != nil { - return &OpError{Op: "set", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// SetNoDelay controls whether the operating system should delay -// packet transmission in hopes of sending fewer packets (Nagle's -// algorithm). The default is true (no delay), meaning that data is -// sent as soon as possible after a Write. -func (c *TCPConn) SetNoDelay(noDelay bool) error { - return &OpError{Op: "set", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} -} - -// DialTCP connects to the remote address raddr on the network net, -// which must be "tcp", "tcp4", or "tcp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialTCP(net string, laddr, raddr *TCPAddr) (*TCPConn, error) { - return dialTCP(net, laddr, raddr, noDeadline, noCancel) + return doDialTCP(ctx, net, laddr, raddr) } -func dialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time, cancel <-chan struct{}) (*TCPConn, error) { - if !deadline.IsZero() { - panic("net.dialTCP: deadline not implemented on Plan 9") - } - // TODO(bradfitz,0intro): also use the cancel channel. +func doDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { switch net { case "tcp", "tcp4", "tcp6": default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + return nil, UnknownNetworkError(net) } if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + return nil, errMissingAddress } - fd, err := dialPlan9(net, laddr, raddr) + fd, err := dialPlan9(ctx, net, laddr, raddr) if err != nil { return nil, err } return newTCPConn(fd), nil } -// TCPListener is a TCP network listener. Clients should typically -// use variables of type Listener instead of assuming TCP. -type TCPListener struct { - fd *netFD -} +func (ln *TCPListener) ok() bool { return ln != nil && ln.fd != nil && ln.fd.ctl != nil } -// AcceptTCP accepts the next incoming call and returns the new -// connection. -func (l *TCPListener) AcceptTCP() (*TCPConn, error) { - if l == nil || l.fd == nil || l.fd.ctl == nil { - return nil, syscall.EINVAL - } - fd, err := l.fd.acceptPlan9() +func (ln *TCPListener) accept() (*TCPConn, error) { + fd, err := ln.fd.acceptPlan9() if err != nil { return nil, err } return newTCPConn(fd), nil } -// Accept implements the Accept method in the Listener interface; it -// waits for the next call and returns a generic Conn. -func (l *TCPListener) Accept() (Conn, error) { - if l == nil || l.fd == nil || l.fd.ctl == nil { - return nil, syscall.EINVAL +func (ln *TCPListener) close() error { + if _, err := ln.fd.ctl.WriteString("hangup"); err != nil { + ln.fd.ctl.Close() + return err } - c, err := l.AcceptTCP() - if err != nil { - return nil, err - } - return c, nil -} - -// Close stops listening on the TCP address. -// Already Accepted connections are not closed. -func (l *TCPListener) Close() error { - if l == nil || l.fd == nil || l.fd.ctl == nil { - return syscall.EINVAL - } - if _, err := l.fd.ctl.WriteString("hangup"); err != nil { - l.fd.ctl.Close() - return &OpError{Op: "close", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: err} - } - err := l.fd.ctl.Close() - if err != nil { - err = &OpError{Op: "close", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: err} - } - return err -} - -// Addr returns the listener's network address, a *TCPAddr. -// The Addr returned is shared by all invocations of Addr, so -// do not modify it. -func (l *TCPListener) Addr() Addr { return l.fd.laddr } - -// SetDeadline sets the deadline associated with the listener. -// A zero time value disables the deadline. -func (l *TCPListener) SetDeadline(t time.Time) error { - if l == nil || l.fd == nil || l.fd.ctl == nil { - return syscall.EINVAL - } - if err := l.fd.setDeadline(t); err != nil { - return &OpError{Op: "set", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: err} + if err := ln.fd.ctl.Close(); err != nil { + return err } return nil } -// File returns a copy of the underlying os.File, set to blocking -// mode. It is the caller's responsibility to close f when finished. -// Closing l does not affect f, and closing f does not affect l. -// -// The returned os.File's file descriptor is different from the -// connection's. Attempting to change properties of the original -// using this duplicate may or may not have the desired effect. -func (l *TCPListener) File() (f *os.File, err error) { - f, err = l.dup() +func (ln *TCPListener) file() (*os.File, error) { + f, err := ln.dup() if err != nil { - err = &OpError{Op: "file", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: err} + return nil, err } - return + return f, nil } -// ListenTCP announces on the TCP address laddr and returns a TCP -// listener. Net must be "tcp", "tcp4", or "tcp6". If laddr has a -// port of 0, ListenTCP will choose an available port. The caller can -// use the Addr method of TCPListener to retrieve the chosen address. -func ListenTCP(net string, laddr *TCPAddr) (*TCPListener, error) { - switch net { - case "tcp", "tcp4", "tcp6": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - laddr = &TCPAddr{} - } - fd, err := listenPlan9(net, laddr) +func listenTCP(ctx context.Context, network string, laddr *TCPAddr) (*TCPListener, error) { + fd, err := listenPlan9(ctx, network, laddr) if err != nil { return nil, err } diff --git a/libgo/go/net/tcpsock_posix.go b/libgo/go/net/tcpsock_posix.go index 0e12d54..c9a8b68 100644 --- a/libgo/go/net/tcpsock_posix.go +++ b/libgo/go/net/tcpsock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,10 +7,10 @@ package net import ( + "context" "io" "os" "syscall" - "time" ) func sockaddrToTCP(sa syscall.Sockaddr) Addr { @@ -40,151 +40,38 @@ func (a *TCPAddr) sockaddr(family int) (syscall.Sockaddr, error) { return ipToSockaddr(family, a.IP, a.Port, a.Zone) } -// TCPConn is an implementation of the Conn interface for TCP network -// connections. -type TCPConn struct { - conn -} - -func newTCPConn(fd *netFD) *TCPConn { - c := &TCPConn{conn{fd}} - setNoDelay(c.fd, true) - return c -} - -// ReadFrom implements the io.ReaderFrom ReadFrom method. -func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { +func (c *TCPConn) readFrom(r io.Reader) (int64, error) { if n, err, handled := sendFile(c.fd, r); handled { - if err != nil && err != io.EOF { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return n, err } - n, err := genericReadFrom(c, r) - if err != nil && err != io.EOF { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return n, err -} - -// CloseRead shuts down the reading side of the TCP connection. -// Most callers should just use Close. -func (c *TCPConn) CloseRead() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeRead() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err -} - -// CloseWrite shuts down the writing side of the TCP connection. -// Most callers should just use Close. -func (c *TCPConn) CloseWrite() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeWrite() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err + return genericReadFrom(c, r) } -// SetLinger sets the behavior of Close on a connection which still -// has data waiting to be sent or to be acknowledged. -// -// If sec < 0 (the default), the operating system finishes sending the -// data in the background. -// -// If sec == 0, the operating system discards any unsent or -// unacknowledged data. -// -// If sec > 0, the data is sent in the background as with sec < 0. On -// some operating systems after sec seconds have elapsed any remaining -// unsent data may be discarded. -func (c *TCPConn) SetLinger(sec int) error { - if !c.ok() { - return syscall.EINVAL - } - if err := setLinger(c.fd, sec); err != nil { - return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} +func dialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + if testHookDialTCP != nil { + return testHookDialTCP(ctx, net, laddr, raddr) } - return nil + return doDialTCP(ctx, net, laddr, raddr) } -// SetKeepAlive sets whether the operating system should send -// keepalive messages on the connection. -func (c *TCPConn) SetKeepAlive(keepalive bool) error { - if !c.ok() { - return syscall.EINVAL - } - if err := setKeepAlive(c.fd, keepalive); err != nil { - return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// SetKeepAlivePeriod sets period between keep alives. -func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { - if !c.ok() { - return syscall.EINVAL - } - if err := setKeepAlivePeriod(c.fd, d); err != nil { - return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// SetNoDelay controls whether the operating system should delay -// packet transmission in hopes of sending fewer packets (Nagle's -// algorithm). The default is true (no delay), meaning that data is -// sent as soon as possible after a Write. -func (c *TCPConn) SetNoDelay(noDelay bool) error { - if !c.ok() { - return syscall.EINVAL - } - if err := setNoDelay(c.fd, noDelay); err != nil { - return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// DialTCP connects to the remote address raddr on the network net, -// which must be "tcp", "tcp4", or "tcp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialTCP(net string, laddr, raddr *TCPAddr) (*TCPConn, error) { - switch net { - case "tcp", "tcp4", "tcp6": - default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} - } - if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} - } - return dialTCP(net, laddr, raddr, noDeadline, noCancel) -} - -func dialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time, cancel <-chan struct{}) (*TCPConn, error) { - fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_STREAM, 0, "dial", cancel) +func doDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + fd, err := internetSocket(ctx, net, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") // TCP has a rarely used mechanism called a 'simultaneous connection' in // which Dial("tcp", addr1, addr2) run on the machine at addr1 can // connect to a simultaneous Dial("tcp", addr2, addr1) run on the machine - // at addr2, without either machine executing Listen. If laddr == nil, + // at addr2, without either machine executing Listen. If laddr == nil, // it means we want the kernel to pick an appropriate originating local - // address. Some Linux kernels cycle blindly through a fixed range of - // local ports, regardless of destination port. If a kernel happens to + // address. Some Linux kernels cycle blindly through a fixed range of + // local ports, regardless of destination port. If a kernel happens to // pick local port 50001 as the source for a Dial("tcp", "", "localhost:50001"), // then the Dial will succeed, having simultaneously connected to itself. // This can only happen when we are letting the kernel pick a port (laddr == nil) // and when there is no listener for the destination address. - // It's hard to argue this is anything other than a kernel bug. If we + // It's hard to argue this is anything other than a kernel bug. If we // see this happen, rather than expose the buggy effect to users, we - // close the fd and try again. If it happens twice more, we relent and - // use the result. See also: + // close the fd and try again. If it happens twice more, we relent and + // use the result. See also: // https://golang.org/issue/2690 // http://stackoverflow.com/questions/4949858/ // @@ -198,11 +85,11 @@ func dialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time, cancel <-cha if err == nil { fd.Close() } - fd, err = internetSocket(net, laddr, raddr, deadline, syscall.SOCK_STREAM, 0, "dial", cancel) + fd, err = internetSocket(ctx, net, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") } if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, err } return newTCPConn(fd), nil } @@ -239,96 +126,32 @@ func spuriousENOTAVAIL(err error) bool { return err == syscall.EADDRNOTAVAIL } -// TCPListener is a TCP network listener. Clients should typically -// use variables of type Listener instead of assuming TCP. -type TCPListener struct { - fd *netFD -} - -// AcceptTCP accepts the next incoming call and returns the new -// connection. -func (l *TCPListener) AcceptTCP() (*TCPConn, error) { - if l == nil || l.fd == nil { - return nil, syscall.EINVAL - } - fd, err := l.fd.accept() - if err != nil { - return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} - } - return newTCPConn(fd), nil -} +func (ln *TCPListener) ok() bool { return ln != nil && ln.fd != nil } -// Accept implements the Accept method in the Listener interface; it -// waits for the next call and returns a generic Conn. -func (l *TCPListener) Accept() (Conn, error) { - c, err := l.AcceptTCP() +func (ln *TCPListener) accept() (*TCPConn, error) { + fd, err := ln.fd.accept() if err != nil { return nil, err } - return c, nil -} - -// Close stops listening on the TCP address. -// Already Accepted connections are not closed. -func (l *TCPListener) Close() error { - if l == nil || l.fd == nil { - return syscall.EINVAL - } - err := l.fd.Close() - if err != nil { - err = &OpError{Op: "close", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} - } - return err + return newTCPConn(fd), nil } -// Addr returns the listener's network address, a *TCPAddr. -// The Addr returned is shared by all invocations of Addr, so -// do not modify it. -func (l *TCPListener) Addr() Addr { return l.fd.laddr } - -// SetDeadline sets the deadline associated with the listener. -// A zero time value disables the deadline. -func (l *TCPListener) SetDeadline(t time.Time) error { - if l == nil || l.fd == nil { - return syscall.EINVAL - } - if err := l.fd.setDeadline(t); err != nil { - return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} - } - return nil +func (ln *TCPListener) close() error { + return ln.fd.Close() } -// File returns a copy of the underlying os.File, set to blocking -// mode. It is the caller's responsibility to close f when finished. -// Closing l does not affect f, and closing f does not affect l. -// -// The returned os.File's file descriptor is different from the -// connection's. Attempting to change properties of the original -// using this duplicate may or may not have the desired effect. -func (l *TCPListener) File() (f *os.File, err error) { - f, err = l.fd.dup() +func (ln *TCPListener) file() (*os.File, error) { + f, err := ln.fd.dup() if err != nil { - err = &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + return nil, err } - return + return f, nil } -// ListenTCP announces on the TCP address laddr and returns a TCP -// listener. Net must be "tcp", "tcp4", or "tcp6". If laddr has a -// port of 0, ListenTCP will choose an available port. The caller can -// use the Addr method of TCPListener to retrieve the chosen address. -func ListenTCP(net string, laddr *TCPAddr) (*TCPListener, error) { - switch net { - case "tcp", "tcp4", "tcp6": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - laddr = &TCPAddr{} - } - fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_STREAM, 0, "listen", noCancel) +func listenTCP(ctx context.Context, network string, laddr *TCPAddr) (*TCPListener, error) { + fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_STREAM, 0, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } return &TCPListener{fd}, nil } diff --git a/libgo/go/net/tcp_test.go b/libgo/go/net/tcpsock_test.go index 9da6f6c..a8d93b0 100644 --- a/libgo/go/net/tcp_test.go +++ b/libgo/go/net/tcpsock_test.go @@ -5,6 +5,7 @@ package net import ( + "internal/testenv" "io" "reflect" "runtime" @@ -345,9 +346,7 @@ var tcpListenerNameTests = []struct { } func TestTCPListenerName(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) for _, tt := range tcpListenerNameTests { ln, err := ListenTCP(tt.net, tt.laddr) @@ -363,9 +362,8 @@ func TestTCPListenerName(t *testing.T) { } func TestIPv6LinkLocalUnicastTCP(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv6 { t.Skip("IPv6 is not supported") } @@ -461,7 +459,6 @@ func TestTCPConcurrentAccept(t *testing.T) { } func TestTCPReadWriteAllocs(t *testing.T) { - t.Skip("skipping test on gccgo until escape analysis is turned on") switch runtime.GOOS { case "nacl", "windows": // NaCl needs to allocate pseudo file descriptor @@ -503,7 +500,8 @@ func TestTCPReadWriteAllocs(t *testing.T) { t.Fatal(err) } }) - if allocs > 0 { + // For gccgo changed "> 0" to "> 7". + if allocs > 7 { t.Fatalf("got %v; want 0", allocs) } } @@ -589,3 +587,50 @@ func TestTCPStress(t *testing.T) { ln.Close() <-done } + +func TestTCPSelfConnect(t *testing.T) { + if runtime.GOOS == "windows" { + // TODO(brainman): do not know why it hangs. + t.Skip("known-broken test on windows") + } + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + var d Dialer + c, err := d.Dial(ln.Addr().Network(), ln.Addr().String()) + if err != nil { + ln.Close() + t.Fatal(err) + } + network := c.LocalAddr().Network() + laddr := *c.LocalAddr().(*TCPAddr) + c.Close() + ln.Close() + + // Try to connect to that address repeatedly. + n := 100000 + if testing.Short() { + n = 1000 + } + switch runtime.GOOS { + case "darwin", "dragonfly", "freebsd", "netbsd", "openbsd", "plan9", "solaris", "windows": + // Non-Linux systems take a long time to figure + // out that there is nothing listening on localhost. + n = 100 + } + for i := 0; i < n; i++ { + d.Timeout = time.Millisecond + c, err := d.Dial(network, laddr.String()) + if err == nil { + addr := c.LocalAddr().(*TCPAddr) + if addr.Port == laddr.Port || addr.IP.Equal(laddr.IP) { + t.Errorf("Dial %v should fail", addr) + } else { + t.Logf("Dial %v succeeded - possibly racing with other listener", addr) + } + c.Close() + } + } +} diff --git a/libgo/go/net/tcpsock_unix_test.go b/libgo/go/net/tcpsock_unix_test.go new file mode 100644 index 0000000..c07f7d7 --- /dev/null +++ b/libgo/go/net/tcpsock_unix_test.go @@ -0,0 +1,79 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +package net + +import ( + "runtime" + "sync" + "syscall" + "testing" + "time" +) + +// See golang.org/issue/14548. +func TestTCPSupriousConnSetupCompletion(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(1) + go func(ln Listener) { + defer wg.Done() + for { + c, err := ln.Accept() + if err != nil { + return + } + wg.Add(1) + go func(c Conn) { + var b [1]byte + c.Read(b[:]) + c.Close() + wg.Done() + }(c) + } + }(ln) + + attempts := int(1e4) // larger is better + wg.Add(attempts) + throttle := make(chan struct{}, runtime.GOMAXPROCS(-1)*2) + for i := 0; i < attempts; i++ { + throttle <- struct{}{} + go func(i int) { + defer func() { + <-throttle + wg.Done() + }() + d := Dialer{Timeout: 50 * time.Millisecond} + c, err := d.Dial(ln.Addr().Network(), ln.Addr().String()) + if err != nil { + if perr := parseDialError(err); perr != nil { + t.Errorf("#%d: %v", i, err) + } + return + } + var b [1]byte + if _, err := c.Write(b[:]); err != nil { + if perr := parseWriteError(err); perr != nil { + t.Errorf("#%d: %v", i, err) + } + if samePlatformError(err, syscall.ENOTCONN) { + t.Errorf("#%d: %v", i, err) + } + } + c.Close() + }(i) + } + + ln.Close() + wg.Wait() +} diff --git a/libgo/go/net/tcpsockopt_darwin.go b/libgo/go/net/tcpsockopt_darwin.go index 1f16090..0d1310e 100644 --- a/libgo/go/net/tcpsockopt_darwin.go +++ b/libgo/go/net/tcpsockopt_darwin.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_dragonfly.go b/libgo/go/net/tcpsockopt_dragonfly.go index 0aa2132..7cc716b 100644 --- a/libgo/go/net/tcpsockopt_dragonfly.go +++ b/libgo/go/net/tcpsockopt_dragonfly.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_openbsd.go b/libgo/go/net/tcpsockopt_openbsd.go index 041e178..10e1bef 100644 --- a/libgo/go/net/tcpsockopt_openbsd.go +++ b/libgo/go/net/tcpsockopt_openbsd.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_plan9.go b/libgo/go/net/tcpsockopt_plan9.go index 157282a..fb56871 100644 --- a/libgo/go/net/tcpsockopt_plan9.go +++ b/libgo/go/net/tcpsockopt_plan9.go @@ -1,4 +1,4 @@ -// Copyright 2014 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,9 +7,14 @@ package net import ( + "syscall" "time" ) +func setNoDelay(fd *netFD, noDelay bool) error { + return syscall.EPLAN9 +} + // Set keep alive period. func setKeepAlivePeriod(fd *netFD, d time.Duration) error { cmd := "keepalive " + itoa(int(d/time.Millisecond)) diff --git a/libgo/go/net/tcpsockopt_posix.go b/libgo/go/net/tcpsockopt_posix.go index 0abf3f9..805b56b 100644 --- a/libgo/go/net/tcpsockopt_posix.go +++ b/libgo/go/net/tcpsockopt_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_solaris.go b/libgo/go/net/tcpsockopt_solaris.go index eaab6b6..347c17d 100644 --- a/libgo/go/net/tcpsockopt_solaris.go +++ b/libgo/go/net/tcpsockopt_solaris.go @@ -1,4 +1,4 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_stub.go b/libgo/go/net/tcpsockopt_stub.go index b413a76..19c83e6 100644 --- a/libgo/go/net/tcpsockopt_stub.go +++ b/libgo/go/net/tcpsockopt_stub.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_unix.go b/libgo/go/net/tcpsockopt_unix.go index c8970d1..8d44fb2 100644 --- a/libgo/go/net/tcpsockopt_unix.go +++ b/libgo/go/net/tcpsockopt_unix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_windows.go b/libgo/go/net/tcpsockopt_windows.go index ae2d7c8..45a4dca 100644 --- a/libgo/go/net/tcpsockopt_windows.go +++ b/libgo/go/net/tcpsockopt_windows.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/testdata/Mark.Twain-Tom.Sawyer.txt b/libgo/go/net/testdata/Mark.Twain-Tom.Sawyer.txt new file mode 100644 index 0000000..c9106fd --- /dev/null +++ b/libgo/go/net/testdata/Mark.Twain-Tom.Sawyer.txt @@ -0,0 +1,8465 @@ +Produced by David Widger. The previous edition was updated by Jose +Menendez. + + + + + + THE ADVENTURES OF TOM SAWYER + BY + MARK TWAIN + (Samuel Langhorne Clemens) + + + + + P R E F A C E + +MOST of the adventures recorded in this book really occurred; one or +two were experiences of my own, the rest those of boys who were +schoolmates of mine. Huck Finn is drawn from life; Tom Sawyer also, but +not from an individual--he is a combination of the characteristics of +three boys whom I knew, and therefore belongs to the composite order of +architecture. + +The odd superstitions touched upon were all prevalent among children +and slaves in the West at the period of this story--that is to say, +thirty or forty years ago. + +Although my book is intended mainly for the entertainment of boys and +girls, I hope it will not be shunned by men and women on that account, +for part of my plan has been to try to pleasantly remind adults of what +they once were themselves, and of how they felt and thought and talked, +and what queer enterprises they sometimes engaged in. + + THE AUTHOR. + +HARTFORD, 1876. + + + + T O M S A W Y E R + + + +CHAPTER I + +"TOM!" + +No answer. + +"TOM!" + +No answer. + +"What's gone with that boy, I wonder? You TOM!" + +No answer. + +The old lady pulled her spectacles down and looked over them about the +room; then she put them up and looked out under them. She seldom or +never looked THROUGH them for so small a thing as a boy; they were her +state pair, the pride of her heart, and were built for "style," not +service--she could have seen through a pair of stove-lids just as well. +She looked perplexed for a moment, and then said, not fiercely, but +still loud enough for the furniture to hear: + +"Well, I lay if I get hold of you I'll--" + +She did not finish, for by this time she was bending down and punching +under the bed with the broom, and so she needed breath to punctuate the +punches with. She resurrected nothing but the cat. + +"I never did see the beat of that boy!" + +She went to the open door and stood in it and looked out among the +tomato vines and "jimpson" weeds that constituted the garden. No Tom. +So she lifted up her voice at an angle calculated for distance and +shouted: + +"Y-o-u-u TOM!" + +There was a slight noise behind her and she turned just in time to +seize a small boy by the slack of his roundabout and arrest his flight. + +"There! I might 'a' thought of that closet. What you been doing in +there?" + +"Nothing." + +"Nothing! Look at your hands. And look at your mouth. What IS that +truck?" + +"I don't know, aunt." + +"Well, I know. It's jam--that's what it is. Forty times I've said if +you didn't let that jam alone I'd skin you. Hand me that switch." + +The switch hovered in the air--the peril was desperate-- + +"My! Look behind you, aunt!" + +The old lady whirled round, and snatched her skirts out of danger. The +lad fled on the instant, scrambled up the high board-fence, and +disappeared over it. + +His aunt Polly stood surprised a moment, and then broke into a gentle +laugh. + +"Hang the boy, can't I never learn anything? Ain't he played me tricks +enough like that for me to be looking out for him by this time? But old +fools is the biggest fools there is. Can't learn an old dog new tricks, +as the saying is. But my goodness, he never plays them alike, two days, +and how is a body to know what's coming? He 'pears to know just how +long he can torment me before I get my dander up, and he knows if he +can make out to put me off for a minute or make me laugh, it's all down +again and I can't hit him a lick. I ain't doing my duty by that boy, +and that's the Lord's truth, goodness knows. Spare the rod and spile +the child, as the Good Book says. I'm a laying up sin and suffering for +us both, I know. He's full of the Old Scratch, but laws-a-me! he's my +own dead sister's boy, poor thing, and I ain't got the heart to lash +him, somehow. Every time I let him off, my conscience does hurt me so, +and every time I hit him my old heart most breaks. Well-a-well, man +that is born of woman is of few days and full of trouble, as the +Scripture says, and I reckon it's so. He'll play hookey this evening, * +and [* Southwestern for "afternoon"] I'll just be obleeged to make him +work, to-morrow, to punish him. It's mighty hard to make him work +Saturdays, when all the boys is having holiday, but he hates work more +than he hates anything else, and I've GOT to do some of my duty by him, +or I'll be the ruination of the child." + +Tom did play hookey, and he had a very good time. He got back home +barely in season to help Jim, the small colored boy, saw next-day's +wood and split the kindlings before supper--at least he was there in +time to tell his adventures to Jim while Jim did three-fourths of the +work. Tom's younger brother (or rather half-brother) Sid was already +through with his part of the work (picking up chips), for he was a +quiet boy, and had no adventurous, troublesome ways. + +While Tom was eating his supper, and stealing sugar as opportunity +offered, Aunt Polly asked him questions that were full of guile, and +very deep--for she wanted to trap him into damaging revealments. Like +many other simple-hearted souls, it was her pet vanity to believe she +was endowed with a talent for dark and mysterious diplomacy, and she +loved to contemplate her most transparent devices as marvels of low +cunning. Said she: + +"Tom, it was middling warm in school, warn't it?" + +"Yes'm." + +"Powerful warm, warn't it?" + +"Yes'm." + +"Didn't you want to go in a-swimming, Tom?" + +A bit of a scare shot through Tom--a touch of uncomfortable suspicion. +He searched Aunt Polly's face, but it told him nothing. So he said: + +"No'm--well, not very much." + +The old lady reached out her hand and felt Tom's shirt, and said: + +"But you ain't too warm now, though." And it flattered her to reflect +that she had discovered that the shirt was dry without anybody knowing +that that was what she had in her mind. But in spite of her, Tom knew +where the wind lay, now. So he forestalled what might be the next move: + +"Some of us pumped on our heads--mine's damp yet. See?" + +Aunt Polly was vexed to think she had overlooked that bit of +circumstantial evidence, and missed a trick. Then she had a new +inspiration: + +"Tom, you didn't have to undo your shirt collar where I sewed it, to +pump on your head, did you? Unbutton your jacket!" + +The trouble vanished out of Tom's face. He opened his jacket. His +shirt collar was securely sewed. + +"Bother! Well, go 'long with you. I'd made sure you'd played hookey +and been a-swimming. But I forgive ye, Tom. I reckon you're a kind of a +singed cat, as the saying is--better'n you look. THIS time." + +She was half sorry her sagacity had miscarried, and half glad that Tom +had stumbled into obedient conduct for once. + +But Sidney said: + +"Well, now, if I didn't think you sewed his collar with white thread, +but it's black." + +"Why, I did sew it with white! Tom!" + +But Tom did not wait for the rest. As he went out at the door he said: + +"Siddy, I'll lick you for that." + +In a safe place Tom examined two large needles which were thrust into +the lapels of his jacket, and had thread bound about them--one needle +carried white thread and the other black. He said: + +"She'd never noticed if it hadn't been for Sid. Confound it! sometimes +she sews it with white, and sometimes she sews it with black. I wish to +geeminy she'd stick to one or t'other--I can't keep the run of 'em. But +I bet you I'll lam Sid for that. I'll learn him!" + +He was not the Model Boy of the village. He knew the model boy very +well though--and loathed him. + +Within two minutes, or even less, he had forgotten all his troubles. +Not because his troubles were one whit less heavy and bitter to him +than a man's are to a man, but because a new and powerful interest bore +them down and drove them out of his mind for the time--just as men's +misfortunes are forgotten in the excitement of new enterprises. This +new interest was a valued novelty in whistling, which he had just +acquired from a negro, and he was suffering to practise it undisturbed. +It consisted in a peculiar bird-like turn, a sort of liquid warble, +produced by touching the tongue to the roof of the mouth at short +intervals in the midst of the music--the reader probably remembers how +to do it, if he has ever been a boy. Diligence and attention soon gave +him the knack of it, and he strode down the street with his mouth full +of harmony and his soul full of gratitude. He felt much as an +astronomer feels who has discovered a new planet--no doubt, as far as +strong, deep, unalloyed pleasure is concerned, the advantage was with +the boy, not the astronomer. + +The summer evenings were long. It was not dark, yet. Presently Tom +checked his whistle. A stranger was before him--a boy a shade larger +than himself. A new-comer of any age or either sex was an impressive +curiosity in the poor little shabby village of St. Petersburg. This boy +was well dressed, too--well dressed on a week-day. This was simply +astounding. His cap was a dainty thing, his close-buttoned blue cloth +roundabout was new and natty, and so were his pantaloons. He had shoes +on--and it was only Friday. He even wore a necktie, a bright bit of +ribbon. He had a citified air about him that ate into Tom's vitals. The +more Tom stared at the splendid marvel, the higher he turned up his +nose at his finery and the shabbier and shabbier his own outfit seemed +to him to grow. Neither boy spoke. If one moved, the other moved--but +only sidewise, in a circle; they kept face to face and eye to eye all +the time. Finally Tom said: + +"I can lick you!" + +"I'd like to see you try it." + +"Well, I can do it." + +"No you can't, either." + +"Yes I can." + +"No you can't." + +"I can." + +"You can't." + +"Can!" + +"Can't!" + +An uncomfortable pause. Then Tom said: + +"What's your name?" + +"'Tisn't any of your business, maybe." + +"Well I 'low I'll MAKE it my business." + +"Well why don't you?" + +"If you say much, I will." + +"Much--much--MUCH. There now." + +"Oh, you think you're mighty smart, DON'T you? I could lick you with +one hand tied behind me, if I wanted to." + +"Well why don't you DO it? You SAY you can do it." + +"Well I WILL, if you fool with me." + +"Oh yes--I've seen whole families in the same fix." + +"Smarty! You think you're SOME, now, DON'T you? Oh, what a hat!" + +"You can lump that hat if you don't like it. I dare you to knock it +off--and anybody that'll take a dare will suck eggs." + +"You're a liar!" + +"You're another." + +"You're a fighting liar and dasn't take it up." + +"Aw--take a walk!" + +"Say--if you give me much more of your sass I'll take and bounce a +rock off'n your head." + +"Oh, of COURSE you will." + +"Well I WILL." + +"Well why don't you DO it then? What do you keep SAYING you will for? +Why don't you DO it? It's because you're afraid." + +"I AIN'T afraid." + +"You are." + +"I ain't." + +"You are." + +Another pause, and more eying and sidling around each other. Presently +they were shoulder to shoulder. Tom said: + +"Get away from here!" + +"Go away yourself!" + +"I won't." + +"I won't either." + +So they stood, each with a foot placed at an angle as a brace, and +both shoving with might and main, and glowering at each other with +hate. But neither could get an advantage. After struggling till both +were hot and flushed, each relaxed his strain with watchful caution, +and Tom said: + +"You're a coward and a pup. I'll tell my big brother on you, and he +can thrash you with his little finger, and I'll make him do it, too." + +"What do I care for your big brother? I've got a brother that's bigger +than he is--and what's more, he can throw him over that fence, too." +[Both brothers were imaginary.] + +"That's a lie." + +"YOUR saying so don't make it so." + +Tom drew a line in the dust with his big toe, and said: + +"I dare you to step over that, and I'll lick you till you can't stand +up. Anybody that'll take a dare will steal sheep." + +The new boy stepped over promptly, and said: + +"Now you said you'd do it, now let's see you do it." + +"Don't you crowd me now; you better look out." + +"Well, you SAID you'd do it--why don't you do it?" + +"By jingo! for two cents I WILL do it." + +The new boy took two broad coppers out of his pocket and held them out +with derision. Tom struck them to the ground. In an instant both boys +were rolling and tumbling in the dirt, gripped together like cats; and +for the space of a minute they tugged and tore at each other's hair and +clothes, punched and scratched each other's nose, and covered +themselves with dust and glory. Presently the confusion took form, and +through the fog of battle Tom appeared, seated astride the new boy, and +pounding him with his fists. "Holler 'nuff!" said he. + +The boy only struggled to free himself. He was crying--mainly from rage. + +"Holler 'nuff!"--and the pounding went on. + +At last the stranger got out a smothered "'Nuff!" and Tom let him up +and said: + +"Now that'll learn you. Better look out who you're fooling with next +time." + +The new boy went off brushing the dust from his clothes, sobbing, +snuffling, and occasionally looking back and shaking his head and +threatening what he would do to Tom the "next time he caught him out." +To which Tom responded with jeers, and started off in high feather, and +as soon as his back was turned the new boy snatched up a stone, threw +it and hit him between the shoulders and then turned tail and ran like +an antelope. Tom chased the traitor home, and thus found out where he +lived. He then held a position at the gate for some time, daring the +enemy to come outside, but the enemy only made faces at him through the +window and declined. At last the enemy's mother appeared, and called +Tom a bad, vicious, vulgar child, and ordered him away. So he went +away; but he said he "'lowed" to "lay" for that boy. + +He got home pretty late that night, and when he climbed cautiously in +at the window, he uncovered an ambuscade, in the person of his aunt; +and when she saw the state his clothes were in her resolution to turn +his Saturday holiday into captivity at hard labor became adamantine in +its firmness. + + + +CHAPTER II + +SATURDAY morning was come, and all the summer world was bright and +fresh, and brimming with life. There was a song in every heart; and if +the heart was young the music issued at the lips. There was cheer in +every face and a spring in every step. The locust-trees were in bloom +and the fragrance of the blossoms filled the air. Cardiff Hill, beyond +the village and above it, was green with vegetation and it lay just far +enough away to seem a Delectable Land, dreamy, reposeful, and inviting. + +Tom appeared on the sidewalk with a bucket of whitewash and a +long-handled brush. He surveyed the fence, and all gladness left him and +a deep melancholy settled down upon his spirit. Thirty yards of board +fence nine feet high. Life to him seemed hollow, and existence but a +burden. Sighing, he dipped his brush and passed it along the topmost +plank; repeated the operation; did it again; compared the insignificant +whitewashed streak with the far-reaching continent of unwhitewashed +fence, and sat down on a tree-box discouraged. Jim came skipping out at +the gate with a tin pail, and singing Buffalo Gals. Bringing water from +the town pump had always been hateful work in Tom's eyes, before, but +now it did not strike him so. He remembered that there was company at +the pump. White, mulatto, and negro boys and girls were always there +waiting their turns, resting, trading playthings, quarrelling, +fighting, skylarking. And he remembered that although the pump was only +a hundred and fifty yards off, Jim never got back with a bucket of +water under an hour--and even then somebody generally had to go after +him. Tom said: + +"Say, Jim, I'll fetch the water if you'll whitewash some." + +Jim shook his head and said: + +"Can't, Mars Tom. Ole missis, she tole me I got to go an' git dis +water an' not stop foolin' roun' wid anybody. She say she spec' Mars +Tom gwine to ax me to whitewash, an' so she tole me go 'long an' 'tend +to my own business--she 'lowed SHE'D 'tend to de whitewashin'." + +"Oh, never you mind what she said, Jim. That's the way she always +talks. Gimme the bucket--I won't be gone only a a minute. SHE won't +ever know." + +"Oh, I dasn't, Mars Tom. Ole missis she'd take an' tar de head off'n +me. 'Deed she would." + +"SHE! She never licks anybody--whacks 'em over the head with her +thimble--and who cares for that, I'd like to know. She talks awful, but +talk don't hurt--anyways it don't if she don't cry. Jim, I'll give you +a marvel. I'll give you a white alley!" + +Jim began to waver. + +"White alley, Jim! And it's a bully taw." + +"My! Dat's a mighty gay marvel, I tell you! But Mars Tom I's powerful +'fraid ole missis--" + +"And besides, if you will I'll show you my sore toe." + +Jim was only human--this attraction was too much for him. He put down +his pail, took the white alley, and bent over the toe with absorbing +interest while the bandage was being unwound. In another moment he was +flying down the street with his pail and a tingling rear, Tom was +whitewashing with vigor, and Aunt Polly was retiring from the field +with a slipper in her hand and triumph in her eye. + +But Tom's energy did not last. He began to think of the fun he had +planned for this day, and his sorrows multiplied. Soon the free boys +would come tripping along on all sorts of delicious expeditions, and +they would make a world of fun of him for having to work--the very +thought of it burnt him like fire. He got out his worldly wealth and +examined it--bits of toys, marbles, and trash; enough to buy an +exchange of WORK, maybe, but not half enough to buy so much as half an +hour of pure freedom. So he returned his straitened means to his +pocket, and gave up the idea of trying to buy the boys. At this dark +and hopeless moment an inspiration burst upon him! Nothing less than a +great, magnificent inspiration. + +He took up his brush and went tranquilly to work. Ben Rogers hove in +sight presently--the very boy, of all boys, whose ridicule he had been +dreading. Ben's gait was the hop-skip-and-jump--proof enough that his +heart was light and his anticipations high. He was eating an apple, and +giving a long, melodious whoop, at intervals, followed by a deep-toned +ding-dong-dong, ding-dong-dong, for he was personating a steamboat. As +he drew near, he slackened speed, took the middle of the street, leaned +far over to starboard and rounded to ponderously and with laborious +pomp and circumstance--for he was personating the Big Missouri, and +considered himself to be drawing nine feet of water. He was boat and +captain and engine-bells combined, so he had to imagine himself +standing on his own hurricane-deck giving the orders and executing them: + +"Stop her, sir! Ting-a-ling-ling!" The headway ran almost out, and he +drew up slowly toward the sidewalk. + +"Ship up to back! Ting-a-ling-ling!" His arms straightened and +stiffened down his sides. + +"Set her back on the stabboard! Ting-a-ling-ling! Chow! ch-chow-wow! +Chow!" His right hand, meantime, describing stately circles--for it was +representing a forty-foot wheel. + +"Let her go back on the labboard! Ting-a-lingling! Chow-ch-chow-chow!" +The left hand began to describe circles. + +"Stop the stabboard! Ting-a-ling-ling! Stop the labboard! Come ahead +on the stabboard! Stop her! Let your outside turn over slow! +Ting-a-ling-ling! Chow-ow-ow! Get out that head-line! LIVELY now! +Come--out with your spring-line--what're you about there! Take a turn +round that stump with the bight of it! Stand by that stage, now--let her +go! Done with the engines, sir! Ting-a-ling-ling! SH'T! S'H'T! SH'T!" +(trying the gauge-cocks). + +Tom went on whitewashing--paid no attention to the steamboat. Ben +stared a moment and then said: "Hi-YI! YOU'RE up a stump, ain't you!" + +No answer. Tom surveyed his last touch with the eye of an artist, then +he gave his brush another gentle sweep and surveyed the result, as +before. Ben ranged up alongside of him. Tom's mouth watered for the +apple, but he stuck to his work. Ben said: + +"Hello, old chap, you got to work, hey?" + +Tom wheeled suddenly and said: + +"Why, it's you, Ben! I warn't noticing." + +"Say--I'm going in a-swimming, I am. Don't you wish you could? But of +course you'd druther WORK--wouldn't you? Course you would!" + +Tom contemplated the boy a bit, and said: + +"What do you call work?" + +"Why, ain't THAT work?" + +Tom resumed his whitewashing, and answered carelessly: + +"Well, maybe it is, and maybe it ain't. All I know, is, it suits Tom +Sawyer." + +"Oh come, now, you don't mean to let on that you LIKE it?" + +The brush continued to move. + +"Like it? Well, I don't see why I oughtn't to like it. Does a boy get +a chance to whitewash a fence every day?" + +That put the thing in a new light. Ben stopped nibbling his apple. Tom +swept his brush daintily back and forth--stepped back to note the +effect--added a touch here and there--criticised the effect again--Ben +watching every move and getting more and more interested, more and more +absorbed. Presently he said: + +"Say, Tom, let ME whitewash a little." + +Tom considered, was about to consent; but he altered his mind: + +"No--no--I reckon it wouldn't hardly do, Ben. You see, Aunt Polly's +awful particular about this fence--right here on the street, you know +--but if it was the back fence I wouldn't mind and SHE wouldn't. Yes, +she's awful particular about this fence; it's got to be done very +careful; I reckon there ain't one boy in a thousand, maybe two +thousand, that can do it the way it's got to be done." + +"No--is that so? Oh come, now--lemme just try. Only just a little--I'd +let YOU, if you was me, Tom." + +"Ben, I'd like to, honest injun; but Aunt Polly--well, Jim wanted to +do it, but she wouldn't let him; Sid wanted to do it, and she wouldn't +let Sid. Now don't you see how I'm fixed? If you was to tackle this +fence and anything was to happen to it--" + +"Oh, shucks, I'll be just as careful. Now lemme try. Say--I'll give +you the core of my apple." + +"Well, here--No, Ben, now don't. I'm afeard--" + +"I'll give you ALL of it!" + +Tom gave up the brush with reluctance in his face, but alacrity in his +heart. And while the late steamer Big Missouri worked and sweated in +the sun, the retired artist sat on a barrel in the shade close by, +dangled his legs, munched his apple, and planned the slaughter of more +innocents. There was no lack of material; boys happened along every +little while; they came to jeer, but remained to whitewash. By the time +Ben was fagged out, Tom had traded the next chance to Billy Fisher for +a kite, in good repair; and when he played out, Johnny Miller bought in +for a dead rat and a string to swing it with--and so on, and so on, +hour after hour. And when the middle of the afternoon came, from being +a poor poverty-stricken boy in the morning, Tom was literally rolling +in wealth. He had besides the things before mentioned, twelve marbles, +part of a jews-harp, a piece of blue bottle-glass to look through, a +spool cannon, a key that wouldn't unlock anything, a fragment of chalk, +a glass stopper of a decanter, a tin soldier, a couple of tadpoles, six +fire-crackers, a kitten with only one eye, a brass doorknob, a +dog-collar--but no dog--the handle of a knife, four pieces of +orange-peel, and a dilapidated old window sash. + +He had had a nice, good, idle time all the while--plenty of company +--and the fence had three coats of whitewash on it! If he hadn't run out +of whitewash he would have bankrupted every boy in the village. + +Tom said to himself that it was not such a hollow world, after all. He +had discovered a great law of human action, without knowing it--namely, +that in order to make a man or a boy covet a thing, it is only +necessary to make the thing difficult to attain. If he had been a great +and wise philosopher, like the writer of this book, he would now have +comprehended that Work consists of whatever a body is OBLIGED to do, +and that Play consists of whatever a body is not obliged to do. And +this would help him to understand why constructing artificial flowers +or performing on a tread-mill is work, while rolling ten-pins or +climbing Mont Blanc is only amusement. There are wealthy gentlemen in +England who drive four-horse passenger-coaches twenty or thirty miles +on a daily line, in the summer, because the privilege costs them +considerable money; but if they were offered wages for the service, +that would turn it into work and then they would resign. + +The boy mused awhile over the substantial change which had taken place +in his worldly circumstances, and then wended toward headquarters to +report. + + + +CHAPTER III + +TOM presented himself before Aunt Polly, who was sitting by an open +window in a pleasant rearward apartment, which was bedroom, +breakfast-room, dining-room, and library, combined. The balmy summer +air, the restful quiet, the odor of the flowers, and the drowsing murmur +of the bees had had their effect, and she was nodding over her knitting +--for she had no company but the cat, and it was asleep in her lap. Her +spectacles were propped up on her gray head for safety. She had thought +that of course Tom had deserted long ago, and she wondered at seeing him +place himself in her power again in this intrepid way. He said: "Mayn't +I go and play now, aunt?" + +"What, a'ready? How much have you done?" + +"It's all done, aunt." + +"Tom, don't lie to me--I can't bear it." + +"I ain't, aunt; it IS all done." + +Aunt Polly placed small trust in such evidence. She went out to see +for herself; and she would have been content to find twenty per cent. +of Tom's statement true. When she found the entire fence whitewashed, +and not only whitewashed but elaborately coated and recoated, and even +a streak added to the ground, her astonishment was almost unspeakable. +She said: + +"Well, I never! There's no getting round it, you can work when you're +a mind to, Tom." And then she diluted the compliment by adding, "But +it's powerful seldom you're a mind to, I'm bound to say. Well, go 'long +and play; but mind you get back some time in a week, or I'll tan you." + +She was so overcome by the splendor of his achievement that she took +him into the closet and selected a choice apple and delivered it to +him, along with an improving lecture upon the added value and flavor a +treat took to itself when it came without sin through virtuous effort. +And while she closed with a happy Scriptural flourish, he "hooked" a +doughnut. + +Then he skipped out, and saw Sid just starting up the outside stairway +that led to the back rooms on the second floor. Clods were handy and +the air was full of them in a twinkling. They raged around Sid like a +hail-storm; and before Aunt Polly could collect her surprised faculties +and sally to the rescue, six or seven clods had taken personal effect, +and Tom was over the fence and gone. There was a gate, but as a general +thing he was too crowded for time to make use of it. His soul was at +peace, now that he had settled with Sid for calling attention to his +black thread and getting him into trouble. + +Tom skirted the block, and came round into a muddy alley that led by +the back of his aunt's cow-stable. He presently got safely beyond the +reach of capture and punishment, and hastened toward the public square +of the village, where two "military" companies of boys had met for +conflict, according to previous appointment. Tom was General of one of +these armies, Joe Harper (a bosom friend) General of the other. These +two great commanders did not condescend to fight in person--that being +better suited to the still smaller fry--but sat together on an eminence +and conducted the field operations by orders delivered through +aides-de-camp. Tom's army won a great victory, after a long and +hard-fought battle. Then the dead were counted, prisoners exchanged, +the terms of the next disagreement agreed upon, and the day for the +necessary battle appointed; after which the armies fell into line and +marched away, and Tom turned homeward alone. + +As he was passing by the house where Jeff Thatcher lived, he saw a new +girl in the garden--a lovely little blue-eyed creature with yellow hair +plaited into two long-tails, white summer frock and embroidered +pantalettes. The fresh-crowned hero fell without firing a shot. A +certain Amy Lawrence vanished out of his heart and left not even a +memory of herself behind. He had thought he loved her to distraction; +he had regarded his passion as adoration; and behold it was only a poor +little evanescent partiality. He had been months winning her; she had +confessed hardly a week ago; he had been the happiest and the proudest +boy in the world only seven short days, and here in one instant of time +she had gone out of his heart like a casual stranger whose visit is +done. + +He worshipped this new angel with furtive eye, till he saw that she +had discovered him; then he pretended he did not know she was present, +and began to "show off" in all sorts of absurd boyish ways, in order to +win her admiration. He kept up this grotesque foolishness for some +time; but by-and-by, while he was in the midst of some dangerous +gymnastic performances, he glanced aside and saw that the little girl +was wending her way toward the house. Tom came up to the fence and +leaned on it, grieving, and hoping she would tarry yet awhile longer. +She halted a moment on the steps and then moved toward the door. Tom +heaved a great sigh as she put her foot on the threshold. But his face +lit up, right away, for she tossed a pansy over the fence a moment +before she disappeared. + +The boy ran around and stopped within a foot or two of the flower, and +then shaded his eyes with his hand and began to look down street as if +he had discovered something of interest going on in that direction. +Presently he picked up a straw and began trying to balance it on his +nose, with his head tilted far back; and as he moved from side to side, +in his efforts, he edged nearer and nearer toward the pansy; finally +his bare foot rested upon it, his pliant toes closed upon it, and he +hopped away with the treasure and disappeared round the corner. But +only for a minute--only while he could button the flower inside his +jacket, next his heart--or next his stomach, possibly, for he was not +much posted in anatomy, and not hypercritical, anyway. + +He returned, now, and hung about the fence till nightfall, "showing +off," as before; but the girl never exhibited herself again, though Tom +comforted himself a little with the hope that she had been near some +window, meantime, and been aware of his attentions. Finally he strode +home reluctantly, with his poor head full of visions. + +All through supper his spirits were so high that his aunt wondered +"what had got into the child." He took a good scolding about clodding +Sid, and did not seem to mind it in the least. He tried to steal sugar +under his aunt's very nose, and got his knuckles rapped for it. He said: + +"Aunt, you don't whack Sid when he takes it." + +"Well, Sid don't torment a body the way you do. You'd be always into +that sugar if I warn't watching you." + +Presently she stepped into the kitchen, and Sid, happy in his +immunity, reached for the sugar-bowl--a sort of glorying over Tom which +was wellnigh unbearable. But Sid's fingers slipped and the bowl dropped +and broke. Tom was in ecstasies. In such ecstasies that he even +controlled his tongue and was silent. He said to himself that he would +not speak a word, even when his aunt came in, but would sit perfectly +still till she asked who did the mischief; and then he would tell, and +there would be nothing so good in the world as to see that pet model +"catch it." He was so brimful of exultation that he could hardly hold +himself when the old lady came back and stood above the wreck +discharging lightnings of wrath from over her spectacles. He said to +himself, "Now it's coming!" And the next instant he was sprawling on +the floor! The potent palm was uplifted to strike again when Tom cried +out: + +"Hold on, now, what 'er you belting ME for?--Sid broke it!" + +Aunt Polly paused, perplexed, and Tom looked for healing pity. But +when she got her tongue again, she only said: + +"Umf! Well, you didn't get a lick amiss, I reckon. You been into some +other audacious mischief when I wasn't around, like enough." + +Then her conscience reproached her, and she yearned to say something +kind and loving; but she judged that this would be construed into a +confession that she had been in the wrong, and discipline forbade that. +So she kept silence, and went about her affairs with a troubled heart. +Tom sulked in a corner and exalted his woes. He knew that in her heart +his aunt was on her knees to him, and he was morosely gratified by the +consciousness of it. He would hang out no signals, he would take notice +of none. He knew that a yearning glance fell upon him, now and then, +through a film of tears, but he refused recognition of it. He pictured +himself lying sick unto death and his aunt bending over him beseeching +one little forgiving word, but he would turn his face to the wall, and +die with that word unsaid. Ah, how would she feel then? And he pictured +himself brought home from the river, dead, with his curls all wet, and +his sore heart at rest. How she would throw herself upon him, and how +her tears would fall like rain, and her lips pray God to give her back +her boy and she would never, never abuse him any more! But he would lie +there cold and white and make no sign--a poor little sufferer, whose +griefs were at an end. He so worked upon his feelings with the pathos +of these dreams, that he had to keep swallowing, he was so like to +choke; and his eyes swam in a blur of water, which overflowed when he +winked, and ran down and trickled from the end of his nose. And such a +luxury to him was this petting of his sorrows, that he could not bear +to have any worldly cheeriness or any grating delight intrude upon it; +it was too sacred for such contact; and so, presently, when his cousin +Mary danced in, all alive with the joy of seeing home again after an +age-long visit of one week to the country, he got up and moved in +clouds and darkness out at one door as she brought song and sunshine in +at the other. + +He wandered far from the accustomed haunts of boys, and sought +desolate places that were in harmony with his spirit. A log raft in the +river invited him, and he seated himself on its outer edge and +contemplated the dreary vastness of the stream, wishing, the while, +that he could only be drowned, all at once and unconsciously, without +undergoing the uncomfortable routine devised by nature. Then he thought +of his flower. He got it out, rumpled and wilted, and it mightily +increased his dismal felicity. He wondered if she would pity him if she +knew? Would she cry, and wish that she had a right to put her arms +around his neck and comfort him? Or would she turn coldly away like all +the hollow world? This picture brought such an agony of pleasurable +suffering that he worked it over and over again in his mind and set it +up in new and varied lights, till he wore it threadbare. At last he +rose up sighing and departed in the darkness. + +About half-past nine or ten o'clock he came along the deserted street +to where the Adored Unknown lived; he paused a moment; no sound fell +upon his listening ear; a candle was casting a dull glow upon the +curtain of a second-story window. Was the sacred presence there? He +climbed the fence, threaded his stealthy way through the plants, till +he stood under that window; he looked up at it long, and with emotion; +then he laid him down on the ground under it, disposing himself upon +his back, with his hands clasped upon his breast and holding his poor +wilted flower. And thus he would die--out in the cold world, with no +shelter over his homeless head, no friendly hand to wipe the +death-damps from his brow, no loving face to bend pityingly over him +when the great agony came. And thus SHE would see him when she looked +out upon the glad morning, and oh! would she drop one little tear upon +his poor, lifeless form, would she heave one little sigh to see a bright +young life so rudely blighted, so untimely cut down? + +The window went up, a maid-servant's discordant voice profaned the +holy calm, and a deluge of water drenched the prone martyr's remains! + +The strangling hero sprang up with a relieving snort. There was a whiz +as of a missile in the air, mingled with the murmur of a curse, a sound +as of shivering glass followed, and a small, vague form went over the +fence and shot away in the gloom. + +Not long after, as Tom, all undressed for bed, was surveying his +drenched garments by the light of a tallow dip, Sid woke up; but if he +had any dim idea of making any "references to allusions," he thought +better of it and held his peace, for there was danger in Tom's eye. + +Tom turned in without the added vexation of prayers, and Sid made +mental note of the omission. + + + +CHAPTER IV + +THE sun rose upon a tranquil world, and beamed down upon the peaceful +village like a benediction. Breakfast over, Aunt Polly had family +worship: it began with a prayer built from the ground up of solid +courses of Scriptural quotations, welded together with a thin mortar of +originality; and from the summit of this she delivered a grim chapter +of the Mosaic Law, as from Sinai. + +Then Tom girded up his loins, so to speak, and went to work to "get +his verses." Sid had learned his lesson days before. Tom bent all his +energies to the memorizing of five verses, and he chose part of the +Sermon on the Mount, because he could find no verses that were shorter. +At the end of half an hour Tom had a vague general idea of his lesson, +but no more, for his mind was traversing the whole field of human +thought, and his hands were busy with distracting recreations. Mary +took his book to hear him recite, and he tried to find his way through +the fog: + +"Blessed are the--a--a--" + +"Poor"-- + +"Yes--poor; blessed are the poor--a--a--" + +"In spirit--" + +"In spirit; blessed are the poor in spirit, for they--they--" + +"THEIRS--" + +"For THEIRS. Blessed are the poor in spirit, for theirs is the kingdom +of heaven. Blessed are they that mourn, for they--they--" + +"Sh--" + +"For they--a--" + +"S, H, A--" + +"For they S, H--Oh, I don't know what it is!" + +"SHALL!" + +"Oh, SHALL! for they shall--for they shall--a--a--shall mourn--a--a-- +blessed are they that shall--they that--a--they that shall mourn, for +they shall--a--shall WHAT? Why don't you tell me, Mary?--what do you +want to be so mean for?" + +"Oh, Tom, you poor thick-headed thing, I'm not teasing you. I wouldn't +do that. You must go and learn it again. Don't you be discouraged, Tom, +you'll manage it--and if you do, I'll give you something ever so nice. +There, now, that's a good boy." + +"All right! What is it, Mary, tell me what it is." + +"Never you mind, Tom. You know if I say it's nice, it is nice." + +"You bet you that's so, Mary. All right, I'll tackle it again." + +And he did "tackle it again"--and under the double pressure of +curiosity and prospective gain he did it with such spirit that he +accomplished a shining success. Mary gave him a brand-new "Barlow" +knife worth twelve and a half cents; and the convulsion of delight that +swept his system shook him to his foundations. True, the knife would +not cut anything, but it was a "sure-enough" Barlow, and there was +inconceivable grandeur in that--though where the Western boys ever got +the idea that such a weapon could possibly be counterfeited to its +injury is an imposing mystery and will always remain so, perhaps. Tom +contrived to scarify the cupboard with it, and was arranging to begin +on the bureau, when he was called off to dress for Sunday-school. + +Mary gave him a tin basin of water and a piece of soap, and he went +outside the door and set the basin on a little bench there; then he +dipped the soap in the water and laid it down; turned up his sleeves; +poured out the water on the ground, gently, and then entered the +kitchen and began to wipe his face diligently on the towel behind the +door. But Mary removed the towel and said: + +"Now ain't you ashamed, Tom. You mustn't be so bad. Water won't hurt +you." + +Tom was a trifle disconcerted. The basin was refilled, and this time +he stood over it a little while, gathering resolution; took in a big +breath and began. When he entered the kitchen presently, with both eyes +shut and groping for the towel with his hands, an honorable testimony +of suds and water was dripping from his face. But when he emerged from +the towel, he was not yet satisfactory, for the clean territory stopped +short at his chin and his jaws, like a mask; below and beyond this line +there was a dark expanse of unirrigated soil that spread downward in +front and backward around his neck. Mary took him in hand, and when she +was done with him he was a man and a brother, without distinction of +color, and his saturated hair was neatly brushed, and its short curls +wrought into a dainty and symmetrical general effect. [He privately +smoothed out the curls, with labor and difficulty, and plastered his +hair close down to his head; for he held curls to be effeminate, and +his own filled his life with bitterness.] Then Mary got out a suit of +his clothing that had been used only on Sundays during two years--they +were simply called his "other clothes"--and so by that we know the +size of his wardrobe. The girl "put him to rights" after he had dressed +himself; she buttoned his neat roundabout up to his chin, turned his +vast shirt collar down over his shoulders, brushed him off and crowned +him with his speckled straw hat. He now looked exceedingly improved and +uncomfortable. He was fully as uncomfortable as he looked; for there +was a restraint about whole clothes and cleanliness that galled him. He +hoped that Mary would forget his shoes, but the hope was blighted; she +coated them thoroughly with tallow, as was the custom, and brought them +out. He lost his temper and said he was always being made to do +everything he didn't want to do. But Mary said, persuasively: + +"Please, Tom--that's a good boy." + +So he got into the shoes snarling. Mary was soon ready, and the three +children set out for Sunday-school--a place that Tom hated with his +whole heart; but Sid and Mary were fond of it. + +Sabbath-school hours were from nine to half-past ten; and then church +service. Two of the children always remained for the sermon +voluntarily, and the other always remained too--for stronger reasons. +The church's high-backed, uncushioned pews would seat about three +hundred persons; the edifice was but a small, plain affair, with a sort +of pine board tree-box on top of it for a steeple. At the door Tom +dropped back a step and accosted a Sunday-dressed comrade: + +"Say, Billy, got a yaller ticket?" + +"Yes." + +"What'll you take for her?" + +"What'll you give?" + +"Piece of lickrish and a fish-hook." + +"Less see 'em." + +Tom exhibited. They were satisfactory, and the property changed hands. +Then Tom traded a couple of white alleys for three red tickets, and +some small trifle or other for a couple of blue ones. He waylaid other +boys as they came, and went on buying tickets of various colors ten or +fifteen minutes longer. He entered the church, now, with a swarm of +clean and noisy boys and girls, proceeded to his seat and started a +quarrel with the first boy that came handy. The teacher, a grave, +elderly man, interfered; then turned his back a moment and Tom pulled a +boy's hair in the next bench, and was absorbed in his book when the boy +turned around; stuck a pin in another boy, presently, in order to hear +him say "Ouch!" and got a new reprimand from his teacher. Tom's whole +class were of a pattern--restless, noisy, and troublesome. When they +came to recite their lessons, not one of them knew his verses +perfectly, but had to be prompted all along. However, they worried +through, and each got his reward--in small blue tickets, each with a +passage of Scripture on it; each blue ticket was pay for two verses of +the recitation. Ten blue tickets equalled a red one, and could be +exchanged for it; ten red tickets equalled a yellow one; for ten yellow +tickets the superintendent gave a very plainly bound Bible (worth forty +cents in those easy times) to the pupil. How many of my readers would +have the industry and application to memorize two thousand verses, even +for a Dore Bible? And yet Mary had acquired two Bibles in this way--it +was the patient work of two years--and a boy of German parentage had +won four or five. He once recited three thousand verses without +stopping; but the strain upon his mental faculties was too great, and +he was little better than an idiot from that day forth--a grievous +misfortune for the school, for on great occasions, before company, the +superintendent (as Tom expressed it) had always made this boy come out +and "spread himself." Only the older pupils managed to keep their +tickets and stick to their tedious work long enough to get a Bible, and +so the delivery of one of these prizes was a rare and noteworthy +circumstance; the successful pupil was so great and conspicuous for +that day that on the spot every scholar's heart was fired with a fresh +ambition that often lasted a couple of weeks. It is possible that Tom's +mental stomach had never really hungered for one of those prizes, but +unquestionably his entire being had for many a day longed for the glory +and the eclat that came with it. + +In due course the superintendent stood up in front of the pulpit, with +a closed hymn-book in his hand and his forefinger inserted between its +leaves, and commanded attention. When a Sunday-school superintendent +makes his customary little speech, a hymn-book in the hand is as +necessary as is the inevitable sheet of music in the hand of a singer +who stands forward on the platform and sings a solo at a concert +--though why, is a mystery: for neither the hymn-book nor the sheet of +music is ever referred to by the sufferer. This superintendent was a +slim creature of thirty-five, with a sandy goatee and short sandy hair; +he wore a stiff standing-collar whose upper edge almost reached his +ears and whose sharp points curved forward abreast the corners of his +mouth--a fence that compelled a straight lookout ahead, and a turning +of the whole body when a side view was required; his chin was propped +on a spreading cravat which was as broad and as long as a bank-note, +and had fringed ends; his boot toes were turned sharply up, in the +fashion of the day, like sleigh-runners--an effect patiently and +laboriously produced by the young men by sitting with their toes +pressed against a wall for hours together. Mr. Walters was very earnest +of mien, and very sincere and honest at heart; and he held sacred +things and places in such reverence, and so separated them from worldly +matters, that unconsciously to himself his Sunday-school voice had +acquired a peculiar intonation which was wholly absent on week-days. He +began after this fashion: + +"Now, children, I want you all to sit up just as straight and pretty +as you can and give me all your attention for a minute or two. There +--that is it. That is the way good little boys and girls should do. I see +one little girl who is looking out of the window--I am afraid she +thinks I am out there somewhere--perhaps up in one of the trees making +a speech to the little birds. [Applausive titter.] I want to tell you +how good it makes me feel to see so many bright, clean little faces +assembled in a place like this, learning to do right and be good." And +so forth and so on. It is not necessary to set down the rest of the +oration. It was of a pattern which does not vary, and so it is familiar +to us all. + +The latter third of the speech was marred by the resumption of fights +and other recreations among certain of the bad boys, and by fidgetings +and whisperings that extended far and wide, washing even to the bases +of isolated and incorruptible rocks like Sid and Mary. But now every +sound ceased suddenly, with the subsidence of Mr. Walters' voice, and +the conclusion of the speech was received with a burst of silent +gratitude. + +A good part of the whispering had been occasioned by an event which +was more or less rare--the entrance of visitors: lawyer Thatcher, +accompanied by a very feeble and aged man; a fine, portly, middle-aged +gentleman with iron-gray hair; and a dignified lady who was doubtless +the latter's wife. The lady was leading a child. Tom had been restless +and full of chafings and repinings; conscience-smitten, too--he could +not meet Amy Lawrence's eye, he could not brook her loving gaze. But +when he saw this small new-comer his soul was all ablaze with bliss in +a moment. The next moment he was "showing off" with all his might +--cuffing boys, pulling hair, making faces--in a word, using every art +that seemed likely to fascinate a girl and win her applause. His +exaltation had but one alloy--the memory of his humiliation in this +angel's garden--and that record in sand was fast washing out, under +the waves of happiness that were sweeping over it now. + +The visitors were given the highest seat of honor, and as soon as Mr. +Walters' speech was finished, he introduced them to the school. The +middle-aged man turned out to be a prodigious personage--no less a one +than the county judge--altogether the most august creation these +children had ever looked upon--and they wondered what kind of material +he was made of--and they half wanted to hear him roar, and were half +afraid he might, too. He was from Constantinople, twelve miles away--so +he had travelled, and seen the world--these very eyes had looked upon +the county court-house--which was said to have a tin roof. The awe +which these reflections inspired was attested by the impressive silence +and the ranks of staring eyes. This was the great Judge Thatcher, +brother of their own lawyer. Jeff Thatcher immediately went forward, to +be familiar with the great man and be envied by the school. It would +have been music to his soul to hear the whisperings: + +"Look at him, Jim! He's a going up there. Say--look! he's a going to +shake hands with him--he IS shaking hands with him! By jings, don't you +wish you was Jeff?" + +Mr. Walters fell to "showing off," with all sorts of official +bustlings and activities, giving orders, delivering judgments, +discharging directions here, there, everywhere that he could find a +target. The librarian "showed off"--running hither and thither with his +arms full of books and making a deal of the splutter and fuss that +insect authority delights in. The young lady teachers "showed off" +--bending sweetly over pupils that were lately being boxed, lifting +pretty warning fingers at bad little boys and patting good ones +lovingly. The young gentlemen teachers "showed off" with small +scoldings and other little displays of authority and fine attention to +discipline--and most of the teachers, of both sexes, found business up +at the library, by the pulpit; and it was business that frequently had +to be done over again two or three times (with much seeming vexation). +The little girls "showed off" in various ways, and the little boys +"showed off" with such diligence that the air was thick with paper wads +and the murmur of scufflings. And above it all the great man sat and +beamed a majestic judicial smile upon all the house, and warmed himself +in the sun of his own grandeur--for he was "showing off," too. + +There was only one thing wanting to make Mr. Walters' ecstasy +complete, and that was a chance to deliver a Bible-prize and exhibit a +prodigy. Several pupils had a few yellow tickets, but none had enough +--he had been around among the star pupils inquiring. He would have given +worlds, now, to have that German lad back again with a sound mind. + +And now at this moment, when hope was dead, Tom Sawyer came forward +with nine yellow tickets, nine red tickets, and ten blue ones, and +demanded a Bible. This was a thunderbolt out of a clear sky. Walters +was not expecting an application from this source for the next ten +years. But there was no getting around it--here were the certified +checks, and they were good for their face. Tom was therefore elevated +to a place with the Judge and the other elect, and the great news was +announced from headquarters. It was the most stunning surprise of the +decade, and so profound was the sensation that it lifted the new hero +up to the judicial one's altitude, and the school had two marvels to +gaze upon in place of one. The boys were all eaten up with envy--but +those that suffered the bitterest pangs were those who perceived too +late that they themselves had contributed to this hated splendor by +trading tickets to Tom for the wealth he had amassed in selling +whitewashing privileges. These despised themselves, as being the dupes +of a wily fraud, a guileful snake in the grass. + +The prize was delivered to Tom with as much effusion as the +superintendent could pump up under the circumstances; but it lacked +somewhat of the true gush, for the poor fellow's instinct taught him +that there was a mystery here that could not well bear the light, +perhaps; it was simply preposterous that this boy had warehoused two +thousand sheaves of Scriptural wisdom on his premises--a dozen would +strain his capacity, without a doubt. + +Amy Lawrence was proud and glad, and she tried to make Tom see it in +her face--but he wouldn't look. She wondered; then she was just a grain +troubled; next a dim suspicion came and went--came again; she watched; +a furtive glance told her worlds--and then her heart broke, and she was +jealous, and angry, and the tears came and she hated everybody. Tom +most of all (she thought). + +Tom was introduced to the Judge; but his tongue was tied, his breath +would hardly come, his heart quaked--partly because of the awful +greatness of the man, but mainly because he was her parent. He would +have liked to fall down and worship him, if it were in the dark. The +Judge put his hand on Tom's head and called him a fine little man, and +asked him what his name was. The boy stammered, gasped, and got it out: + +"Tom." + +"Oh, no, not Tom--it is--" + +"Thomas." + +"Ah, that's it. I thought there was more to it, maybe. That's very +well. But you've another one I daresay, and you'll tell it to me, won't +you?" + +"Tell the gentleman your other name, Thomas," said Walters, "and say +sir. You mustn't forget your manners." + +"Thomas Sawyer--sir." + +"That's it! That's a good boy. Fine boy. Fine, manly little fellow. +Two thousand verses is a great many--very, very great many. And you +never can be sorry for the trouble you took to learn them; for +knowledge is worth more than anything there is in the world; it's what +makes great men and good men; you'll be a great man and a good man +yourself, some day, Thomas, and then you'll look back and say, It's all +owing to the precious Sunday-school privileges of my boyhood--it's all +owing to my dear teachers that taught me to learn--it's all owing to +the good superintendent, who encouraged me, and watched over me, and +gave me a beautiful Bible--a splendid elegant Bible--to keep and have +it all for my own, always--it's all owing to right bringing up! That is +what you will say, Thomas--and you wouldn't take any money for those +two thousand verses--no indeed you wouldn't. And now you wouldn't mind +telling me and this lady some of the things you've learned--no, I know +you wouldn't--for we are proud of little boys that learn. Now, no +doubt you know the names of all the twelve disciples. Won't you tell us +the names of the first two that were appointed?" + +Tom was tugging at a button-hole and looking sheepish. He blushed, +now, and his eyes fell. Mr. Walters' heart sank within him. He said to +himself, it is not possible that the boy can answer the simplest +question--why DID the Judge ask him? Yet he felt obliged to speak up +and say: + +"Answer the gentleman, Thomas--don't be afraid." + +Tom still hung fire. + +"Now I know you'll tell me," said the lady. "The names of the first +two disciples were--" + +"DAVID AND GOLIAH!" + +Let us draw the curtain of charity over the rest of the scene. + + + +CHAPTER V + +ABOUT half-past ten the cracked bell of the small church began to +ring, and presently the people began to gather for the morning sermon. +The Sunday-school children distributed themselves about the house and +occupied pews with their parents, so as to be under supervision. Aunt +Polly came, and Tom and Sid and Mary sat with her--Tom being placed +next the aisle, in order that he might be as far away from the open +window and the seductive outside summer scenes as possible. The crowd +filed up the aisles: the aged and needy postmaster, who had seen better +days; the mayor and his wife--for they had a mayor there, among other +unnecessaries; the justice of the peace; the widow Douglass, fair, +smart, and forty, a generous, good-hearted soul and well-to-do, her +hill mansion the only palace in the town, and the most hospitable and +much the most lavish in the matter of festivities that St. Petersburg +could boast; the bent and venerable Major and Mrs. Ward; lawyer +Riverson, the new notable from a distance; next the belle of the +village, followed by a troop of lawn-clad and ribbon-decked young +heart-breakers; then all the young clerks in town in a body--for they +had stood in the vestibule sucking their cane-heads, a circling wall of +oiled and simpering admirers, till the last girl had run their gantlet; +and last of all came the Model Boy, Willie Mufferson, taking as heedful +care of his mother as if she were cut glass. He always brought his +mother to church, and was the pride of all the matrons. The boys all +hated him, he was so good. And besides, he had been "thrown up to them" +so much. His white handkerchief was hanging out of his pocket behind, as +usual on Sundays--accidentally. Tom had no handkerchief, and he looked +upon boys who had as snobs. + +The congregation being fully assembled, now, the bell rang once more, +to warn laggards and stragglers, and then a solemn hush fell upon the +church which was only broken by the tittering and whispering of the +choir in the gallery. The choir always tittered and whispered all +through service. There was once a church choir that was not ill-bred, +but I have forgotten where it was, now. It was a great many years ago, +and I can scarcely remember anything about it, but I think it was in +some foreign country. + +The minister gave out the hymn, and read it through with a relish, in +a peculiar style which was much admired in that part of the country. +His voice began on a medium key and climbed steadily up till it reached +a certain point, where it bore with strong emphasis upon the topmost +word and then plunged down as if from a spring-board: + + Shall I be car-ri-ed toe the skies, on flow'ry BEDS of ease, + + Whilst others fight to win the prize, and sail thro' BLOODY seas? + +He was regarded as a wonderful reader. At church "sociables" he was +always called upon to read poetry; and when he was through, the ladies +would lift up their hands and let them fall helplessly in their laps, +and "wall" their eyes, and shake their heads, as much as to say, "Words +cannot express it; it is too beautiful, TOO beautiful for this mortal +earth." + +After the hymn had been sung, the Rev. Mr. Sprague turned himself into +a bulletin-board, and read off "notices" of meetings and societies and +things till it seemed that the list would stretch out to the crack of +doom--a queer custom which is still kept up in America, even in cities, +away here in this age of abundant newspapers. Often, the less there is +to justify a traditional custom, the harder it is to get rid of it. + +And now the minister prayed. A good, generous prayer it was, and went +into details: it pleaded for the church, and the little children of the +church; for the other churches of the village; for the village itself; +for the county; for the State; for the State officers; for the United +States; for the churches of the United States; for Congress; for the +President; for the officers of the Government; for poor sailors, tossed +by stormy seas; for the oppressed millions groaning under the heel of +European monarchies and Oriental despotisms; for such as have the light +and the good tidings, and yet have not eyes to see nor ears to hear +withal; for the heathen in the far islands of the sea; and closed with +a supplication that the words he was about to speak might find grace +and favor, and be as seed sown in fertile ground, yielding in time a +grateful harvest of good. Amen. + +There was a rustling of dresses, and the standing congregation sat +down. The boy whose history this book relates did not enjoy the prayer, +he only endured it--if he even did that much. He was restive all +through it; he kept tally of the details of the prayer, unconsciously +--for he was not listening, but he knew the ground of old, and the +clergyman's regular route over it--and when a little trifle of new +matter was interlarded, his ear detected it and his whole nature +resented it; he considered additions unfair, and scoundrelly. In the +midst of the prayer a fly had lit on the back of the pew in front of +him and tortured his spirit by calmly rubbing its hands together, +embracing its head with its arms, and polishing it so vigorously that +it seemed to almost part company with the body, and the slender thread +of a neck was exposed to view; scraping its wings with its hind legs +and smoothing them to its body as if they had been coat-tails; going +through its whole toilet as tranquilly as if it knew it was perfectly +safe. As indeed it was; for as sorely as Tom's hands itched to grab for +it they did not dare--he believed his soul would be instantly destroyed +if he did such a thing while the prayer was going on. But with the +closing sentence his hand began to curve and steal forward; and the +instant the "Amen" was out the fly was a prisoner of war. His aunt +detected the act and made him let it go. + +The minister gave out his text and droned along monotonously through +an argument that was so prosy that many a head by and by began to nod +--and yet it was an argument that dealt in limitless fire and brimstone +and thinned the predestined elect down to a company so small as to be +hardly worth the saving. Tom counted the pages of the sermon; after +church he always knew how many pages there had been, but he seldom knew +anything else about the discourse. However, this time he was really +interested for a little while. The minister made a grand and moving +picture of the assembling together of the world's hosts at the +millennium when the lion and the lamb should lie down together and a +little child should lead them. But the pathos, the lesson, the moral of +the great spectacle were lost upon the boy; he only thought of the +conspicuousness of the principal character before the on-looking +nations; his face lit with the thought, and he said to himself that he +wished he could be that child, if it was a tame lion. + +Now he lapsed into suffering again, as the dry argument was resumed. +Presently he bethought him of a treasure he had and got it out. It was +a large black beetle with formidable jaws--a "pinchbug," he called it. +It was in a percussion-cap box. The first thing the beetle did was to +take him by the finger. A natural fillip followed, the beetle went +floundering into the aisle and lit on its back, and the hurt finger +went into the boy's mouth. The beetle lay there working its helpless +legs, unable to turn over. Tom eyed it, and longed for it; but it was +safe out of his reach. Other people uninterested in the sermon found +relief in the beetle, and they eyed it too. Presently a vagrant poodle +dog came idling along, sad at heart, lazy with the summer softness and +the quiet, weary of captivity, sighing for change. He spied the beetle; +the drooping tail lifted and wagged. He surveyed the prize; walked +around it; smelt at it from a safe distance; walked around it again; +grew bolder, and took a closer smell; then lifted his lip and made a +gingerly snatch at it, just missing it; made another, and another; +began to enjoy the diversion; subsided to his stomach with the beetle +between his paws, and continued his experiments; grew weary at last, +and then indifferent and absent-minded. His head nodded, and little by +little his chin descended and touched the enemy, who seized it. There +was a sharp yelp, a flirt of the poodle's head, and the beetle fell a +couple of yards away, and lit on its back once more. The neighboring +spectators shook with a gentle inward joy, several faces went behind +fans and handkerchiefs, and Tom was entirely happy. The dog looked +foolish, and probably felt so; but there was resentment in his heart, +too, and a craving for revenge. So he went to the beetle and began a +wary attack on it again; jumping at it from every point of a circle, +lighting with his fore-paws within an inch of the creature, making even +closer snatches at it with his teeth, and jerking his head till his +ears flapped again. But he grew tired once more, after a while; tried +to amuse himself with a fly but found no relief; followed an ant +around, with his nose close to the floor, and quickly wearied of that; +yawned, sighed, forgot the beetle entirely, and sat down on it. Then +there was a wild yelp of agony and the poodle went sailing up the +aisle; the yelps continued, and so did the dog; he crossed the house in +front of the altar; he flew down the other aisle; he crossed before the +doors; he clamored up the home-stretch; his anguish grew with his +progress, till presently he was but a woolly comet moving in its orbit +with the gleam and the speed of light. At last the frantic sufferer +sheered from its course, and sprang into its master's lap; he flung it +out of the window, and the voice of distress quickly thinned away and +died in the distance. + +By this time the whole church was red-faced and suffocating with +suppressed laughter, and the sermon had come to a dead standstill. The +discourse was resumed presently, but it went lame and halting, all +possibility of impressiveness being at an end; for even the gravest +sentiments were constantly being received with a smothered burst of +unholy mirth, under cover of some remote pew-back, as if the poor +parson had said a rarely facetious thing. It was a genuine relief to +the whole congregation when the ordeal was over and the benediction +pronounced. + +Tom Sawyer went home quite cheerful, thinking to himself that there +was some satisfaction about divine service when there was a bit of +variety in it. He had but one marring thought; he was willing that the +dog should play with his pinchbug, but he did not think it was upright +in him to carry it off. + + + +CHAPTER VI + +MONDAY morning found Tom Sawyer miserable. Monday morning always found +him so--because it began another week's slow suffering in school. He +generally began that day with wishing he had had no intervening +holiday, it made the going into captivity and fetters again so much +more odious. + +Tom lay thinking. Presently it occurred to him that he wished he was +sick; then he could stay home from school. Here was a vague +possibility. He canvassed his system. No ailment was found, and he +investigated again. This time he thought he could detect colicky +symptoms, and he began to encourage them with considerable hope. But +they soon grew feeble, and presently died wholly away. He reflected +further. Suddenly he discovered something. One of his upper front teeth +was loose. This was lucky; he was about to begin to groan, as a +"starter," as he called it, when it occurred to him that if he came +into court with that argument, his aunt would pull it out, and that +would hurt. So he thought he would hold the tooth in reserve for the +present, and seek further. Nothing offered for some little time, and +then he remembered hearing the doctor tell about a certain thing that +laid up a patient for two or three weeks and threatened to make him +lose a finger. So the boy eagerly drew his sore toe from under the +sheet and held it up for inspection. But now he did not know the +necessary symptoms. However, it seemed well worth while to chance it, +so he fell to groaning with considerable spirit. + +But Sid slept on unconscious. + +Tom groaned louder, and fancied that he began to feel pain in the toe. + +No result from Sid. + +Tom was panting with his exertions by this time. He took a rest and +then swelled himself up and fetched a succession of admirable groans. + +Sid snored on. + +Tom was aggravated. He said, "Sid, Sid!" and shook him. This course +worked well, and Tom began to groan again. Sid yawned, stretched, then +brought himself up on his elbow with a snort, and began to stare at +Tom. Tom went on groaning. Sid said: + +"Tom! Say, Tom!" [No response.] "Here, Tom! TOM! What is the matter, +Tom?" And he shook him and looked in his face anxiously. + +Tom moaned out: + +"Oh, don't, Sid. Don't joggle me." + +"Why, what's the matter, Tom? I must call auntie." + +"No--never mind. It'll be over by and by, maybe. Don't call anybody." + +"But I must! DON'T groan so, Tom, it's awful. How long you been this +way?" + +"Hours. Ouch! Oh, don't stir so, Sid, you'll kill me." + +"Tom, why didn't you wake me sooner? Oh, Tom, DON'T! It makes my +flesh crawl to hear you. Tom, what is the matter?" + +"I forgive you everything, Sid. [Groan.] Everything you've ever done +to me. When I'm gone--" + +"Oh, Tom, you ain't dying, are you? Don't, Tom--oh, don't. Maybe--" + +"I forgive everybody, Sid. [Groan.] Tell 'em so, Sid. And Sid, you +give my window-sash and my cat with one eye to that new girl that's +come to town, and tell her--" + +But Sid had snatched his clothes and gone. Tom was suffering in +reality, now, so handsomely was his imagination working, and so his +groans had gathered quite a genuine tone. + +Sid flew down-stairs and said: + +"Oh, Aunt Polly, come! Tom's dying!" + +"Dying!" + +"Yes'm. Don't wait--come quick!" + +"Rubbage! I don't believe it!" + +But she fled up-stairs, nevertheless, with Sid and Mary at her heels. +And her face grew white, too, and her lip trembled. When she reached +the bedside she gasped out: + +"You, Tom! Tom, what's the matter with you?" + +"Oh, auntie, I'm--" + +"What's the matter with you--what is the matter with you, child?" + +"Oh, auntie, my sore toe's mortified!" + +The old lady sank down into a chair and laughed a little, then cried a +little, then did both together. This restored her and she said: + +"Tom, what a turn you did give me. Now you shut up that nonsense and +climb out of this." + +The groans ceased and the pain vanished from the toe. The boy felt a +little foolish, and he said: + +"Aunt Polly, it SEEMED mortified, and it hurt so I never minded my +tooth at all." + +"Your tooth, indeed! What's the matter with your tooth?" + +"One of them's loose, and it aches perfectly awful." + +"There, there, now, don't begin that groaning again. Open your mouth. +Well--your tooth IS loose, but you're not going to die about that. +Mary, get me a silk thread, and a chunk of fire out of the kitchen." + +Tom said: + +"Oh, please, auntie, don't pull it out. It don't hurt any more. I wish +I may never stir if it does. Please don't, auntie. I don't want to stay +home from school." + +"Oh, you don't, don't you? So all this row was because you thought +you'd get to stay home from school and go a-fishing? Tom, Tom, I love +you so, and you seem to try every way you can to break my old heart +with your outrageousness." By this time the dental instruments were +ready. The old lady made one end of the silk thread fast to Tom's tooth +with a loop and tied the other to the bedpost. Then she seized the +chunk of fire and suddenly thrust it almost into the boy's face. The +tooth hung dangling by the bedpost, now. + +But all trials bring their compensations. As Tom wended to school +after breakfast, he was the envy of every boy he met because the gap in +his upper row of teeth enabled him to expectorate in a new and +admirable way. He gathered quite a following of lads interested in the +exhibition; and one that had cut his finger and had been a centre of +fascination and homage up to this time, now found himself suddenly +without an adherent, and shorn of his glory. His heart was heavy, and +he said with a disdain which he did not feel that it wasn't anything to +spit like Tom Sawyer; but another boy said, "Sour grapes!" and he +wandered away a dismantled hero. + +Shortly Tom came upon the juvenile pariah of the village, Huckleberry +Finn, son of the town drunkard. Huckleberry was cordially hated and +dreaded by all the mothers of the town, because he was idle and lawless +and vulgar and bad--and because all their children admired him so, and +delighted in his forbidden society, and wished they dared to be like +him. Tom was like the rest of the respectable boys, in that he envied +Huckleberry his gaudy outcast condition, and was under strict orders +not to play with him. So he played with him every time he got a chance. +Huckleberry was always dressed in the cast-off clothes of full-grown +men, and they were in perennial bloom and fluttering with rags. His hat +was a vast ruin with a wide crescent lopped out of its brim; his coat, +when he wore one, hung nearly to his heels and had the rearward buttons +far down the back; but one suspender supported his trousers; the seat +of the trousers bagged low and contained nothing, the fringed legs +dragged in the dirt when not rolled up. + +Huckleberry came and went, at his own free will. He slept on doorsteps +in fine weather and in empty hogsheads in wet; he did not have to go to +school or to church, or call any being master or obey anybody; he could +go fishing or swimming when and where he chose, and stay as long as it +suited him; nobody forbade him to fight; he could sit up as late as he +pleased; he was always the first boy that went barefoot in the spring +and the last to resume leather in the fall; he never had to wash, nor +put on clean clothes; he could swear wonderfully. In a word, everything +that goes to make life precious that boy had. So thought every +harassed, hampered, respectable boy in St. Petersburg. + +Tom hailed the romantic outcast: + +"Hello, Huckleberry!" + +"Hello yourself, and see how you like it." + +"What's that you got?" + +"Dead cat." + +"Lemme see him, Huck. My, he's pretty stiff. Where'd you get him?" + +"Bought him off'n a boy." + +"What did you give?" + +"I give a blue ticket and a bladder that I got at the slaughter-house." + +"Where'd you get the blue ticket?" + +"Bought it off'n Ben Rogers two weeks ago for a hoop-stick." + +"Say--what is dead cats good for, Huck?" + +"Good for? Cure warts with." + +"No! Is that so? I know something that's better." + +"I bet you don't. What is it?" + +"Why, spunk-water." + +"Spunk-water! I wouldn't give a dern for spunk-water." + +"You wouldn't, wouldn't you? D'you ever try it?" + +"No, I hain't. But Bob Tanner did." + +"Who told you so!" + +"Why, he told Jeff Thatcher, and Jeff told Johnny Baker, and Johnny +told Jim Hollis, and Jim told Ben Rogers, and Ben told a nigger, and +the nigger told me. There now!" + +"Well, what of it? They'll all lie. Leastways all but the nigger. I +don't know HIM. But I never see a nigger that WOULDN'T lie. Shucks! Now +you tell me how Bob Tanner done it, Huck." + +"Why, he took and dipped his hand in a rotten stump where the +rain-water was." + +"In the daytime?" + +"Certainly." + +"With his face to the stump?" + +"Yes. Least I reckon so." + +"Did he say anything?" + +"I don't reckon he did. I don't know." + +"Aha! Talk about trying to cure warts with spunk-water such a blame +fool way as that! Why, that ain't a-going to do any good. You got to go +all by yourself, to the middle of the woods, where you know there's a +spunk-water stump, and just as it's midnight you back up against the +stump and jam your hand in and say: + + 'Barley-corn, barley-corn, injun-meal shorts, + Spunk-water, spunk-water, swaller these warts,' + +and then walk away quick, eleven steps, with your eyes shut, and then +turn around three times and walk home without speaking to anybody. +Because if you speak the charm's busted." + +"Well, that sounds like a good way; but that ain't the way Bob Tanner +done." + +"No, sir, you can bet he didn't, becuz he's the wartiest boy in this +town; and he wouldn't have a wart on him if he'd knowed how to work +spunk-water. I've took off thousands of warts off of my hands that way, +Huck. I play with frogs so much that I've always got considerable many +warts. Sometimes I take 'em off with a bean." + +"Yes, bean's good. I've done that." + +"Have you? What's your way?" + +"You take and split the bean, and cut the wart so as to get some +blood, and then you put the blood on one piece of the bean and take and +dig a hole and bury it 'bout midnight at the crossroads in the dark of +the moon, and then you burn up the rest of the bean. You see that piece +that's got the blood on it will keep drawing and drawing, trying to +fetch the other piece to it, and so that helps the blood to draw the +wart, and pretty soon off she comes." + +"Yes, that's it, Huck--that's it; though when you're burying it if you +say 'Down bean; off wart; come no more to bother me!' it's better. +That's the way Joe Harper does, and he's been nearly to Coonville and +most everywheres. But say--how do you cure 'em with dead cats?" + +"Why, you take your cat and go and get in the graveyard 'long about +midnight when somebody that was wicked has been buried; and when it's +midnight a devil will come, or maybe two or three, but you can't see +'em, you can only hear something like the wind, or maybe hear 'em talk; +and when they're taking that feller away, you heave your cat after 'em +and say, 'Devil follow corpse, cat follow devil, warts follow cat, I'm +done with ye!' That'll fetch ANY wart." + +"Sounds right. D'you ever try it, Huck?" + +"No, but old Mother Hopkins told me." + +"Well, I reckon it's so, then. Becuz they say she's a witch." + +"Say! Why, Tom, I KNOW she is. She witched pap. Pap says so his own +self. He come along one day, and he see she was a-witching him, so he +took up a rock, and if she hadn't dodged, he'd a got her. Well, that +very night he rolled off'n a shed wher' he was a layin drunk, and broke +his arm." + +"Why, that's awful. How did he know she was a-witching him?" + +"Lord, pap can tell, easy. Pap says when they keep looking at you +right stiddy, they're a-witching you. Specially if they mumble. Becuz +when they mumble they're saying the Lord's Prayer backards." + +"Say, Hucky, when you going to try the cat?" + +"To-night. I reckon they'll come after old Hoss Williams to-night." + +"But they buried him Saturday. Didn't they get him Saturday night?" + +"Why, how you talk! How could their charms work till midnight?--and +THEN it's Sunday. Devils don't slosh around much of a Sunday, I don't +reckon." + +"I never thought of that. That's so. Lemme go with you?" + +"Of course--if you ain't afeard." + +"Afeard! 'Tain't likely. Will you meow?" + +"Yes--and you meow back, if you get a chance. Last time, you kep' me +a-meowing around till old Hays went to throwing rocks at me and says +'Dern that cat!' and so I hove a brick through his window--but don't +you tell." + +"I won't. I couldn't meow that night, becuz auntie was watching me, +but I'll meow this time. Say--what's that?" + +"Nothing but a tick." + +"Where'd you get him?" + +"Out in the woods." + +"What'll you take for him?" + +"I don't know. I don't want to sell him." + +"All right. It's a mighty small tick, anyway." + +"Oh, anybody can run a tick down that don't belong to them. I'm +satisfied with it. It's a good enough tick for me." + +"Sho, there's ticks a plenty. I could have a thousand of 'em if I +wanted to." + +"Well, why don't you? Becuz you know mighty well you can't. This is a +pretty early tick, I reckon. It's the first one I've seen this year." + +"Say, Huck--I'll give you my tooth for him." + +"Less see it." + +Tom got out a bit of paper and carefully unrolled it. Huckleberry +viewed it wistfully. The temptation was very strong. At last he said: + +"Is it genuwyne?" + +Tom lifted his lip and showed the vacancy. + +"Well, all right," said Huckleberry, "it's a trade." + +Tom enclosed the tick in the percussion-cap box that had lately been +the pinchbug's prison, and the boys separated, each feeling wealthier +than before. + +When Tom reached the little isolated frame schoolhouse, he strode in +briskly, with the manner of one who had come with all honest speed. +He hung his hat on a peg and flung himself into his seat with +business-like alacrity. The master, throned on high in his great +splint-bottom arm-chair, was dozing, lulled by the drowsy hum of study. +The interruption roused him. + +"Thomas Sawyer!" + +Tom knew that when his name was pronounced in full, it meant trouble. + +"Sir!" + +"Come up here. Now, sir, why are you late again, as usual?" + +Tom was about to take refuge in a lie, when he saw two long tails of +yellow hair hanging down a back that he recognized by the electric +sympathy of love; and by that form was THE ONLY VACANT PLACE on the +girls' side of the schoolhouse. He instantly said: + +"I STOPPED TO TALK WITH HUCKLEBERRY FINN!" + +The master's pulse stood still, and he stared helplessly. The buzz of +study ceased. The pupils wondered if this foolhardy boy had lost his +mind. The master said: + +"You--you did what?" + +"Stopped to talk with Huckleberry Finn." + +There was no mistaking the words. + +"Thomas Sawyer, this is the most astounding confession I have ever +listened to. No mere ferule will answer for this offence. Take off your +jacket." + +The master's arm performed until it was tired and the stock of +switches notably diminished. Then the order followed: + +"Now, sir, go and sit with the girls! And let this be a warning to you." + +The titter that rippled around the room appeared to abash the boy, but +in reality that result was caused rather more by his worshipful awe of +his unknown idol and the dread pleasure that lay in his high good +fortune. He sat down upon the end of the pine bench and the girl +hitched herself away from him with a toss of her head. Nudges and winks +and whispers traversed the room, but Tom sat still, with his arms upon +the long, low desk before him, and seemed to study his book. + +By and by attention ceased from him, and the accustomed school murmur +rose upon the dull air once more. Presently the boy began to steal +furtive glances at the girl. She observed it, "made a mouth" at him and +gave him the back of her head for the space of a minute. When she +cautiously faced around again, a peach lay before her. She thrust it +away. Tom gently put it back. She thrust it away again, but with less +animosity. Tom patiently returned it to its place. Then she let it +remain. Tom scrawled on his slate, "Please take it--I got more." The +girl glanced at the words, but made no sign. Now the boy began to draw +something on the slate, hiding his work with his left hand. For a time +the girl refused to notice; but her human curiosity presently began to +manifest itself by hardly perceptible signs. The boy worked on, +apparently unconscious. The girl made a sort of noncommittal attempt to +see, but the boy did not betray that he was aware of it. At last she +gave in and hesitatingly whispered: + +"Let me see it." + +Tom partly uncovered a dismal caricature of a house with two gable +ends to it and a corkscrew of smoke issuing from the chimney. Then the +girl's interest began to fasten itself upon the work and she forgot +everything else. When it was finished, she gazed a moment, then +whispered: + +"It's nice--make a man." + +The artist erected a man in the front yard, that resembled a derrick. +He could have stepped over the house; but the girl was not +hypercritical; she was satisfied with the monster, and whispered: + +"It's a beautiful man--now make me coming along." + +Tom drew an hour-glass with a full moon and straw limbs to it and +armed the spreading fingers with a portentous fan. The girl said: + +"It's ever so nice--I wish I could draw." + +"It's easy," whispered Tom, "I'll learn you." + +"Oh, will you? When?" + +"At noon. Do you go home to dinner?" + +"I'll stay if you will." + +"Good--that's a whack. What's your name?" + +"Becky Thatcher. What's yours? Oh, I know. It's Thomas Sawyer." + +"That's the name they lick me by. I'm Tom when I'm good. You call me +Tom, will you?" + +"Yes." + +Now Tom began to scrawl something on the slate, hiding the words from +the girl. But she was not backward this time. She begged to see. Tom +said: + +"Oh, it ain't anything." + +"Yes it is." + +"No it ain't. You don't want to see." + +"Yes I do, indeed I do. Please let me." + +"You'll tell." + +"No I won't--deed and deed and double deed won't." + +"You won't tell anybody at all? Ever, as long as you live?" + +"No, I won't ever tell ANYbody. Now let me." + +"Oh, YOU don't want to see!" + +"Now that you treat me so, I WILL see." And she put her small hand +upon his and a little scuffle ensued, Tom pretending to resist in +earnest but letting his hand slip by degrees till these words were +revealed: "I LOVE YOU." + +"Oh, you bad thing!" And she hit his hand a smart rap, but reddened +and looked pleased, nevertheless. + +Just at this juncture the boy felt a slow, fateful grip closing on his +ear, and a steady lifting impulse. In that wise he was borne across the +house and deposited in his own seat, under a peppering fire of giggles +from the whole school. Then the master stood over him during a few +awful moments, and finally moved away to his throne without saying a +word. But although Tom's ear tingled, his heart was jubilant. + +As the school quieted down Tom made an honest effort to study, but the +turmoil within him was too great. In turn he took his place in the +reading class and made a botch of it; then in the geography class and +turned lakes into mountains, mountains into rivers, and rivers into +continents, till chaos was come again; then in the spelling class, and +got "turned down," by a succession of mere baby words, till he brought +up at the foot and yielded up the pewter medal which he had worn with +ostentation for months. + + + +CHAPTER VII + +THE harder Tom tried to fasten his mind on his book, the more his +ideas wandered. So at last, with a sigh and a yawn, he gave it up. It +seemed to him that the noon recess would never come. The air was +utterly dead. There was not a breath stirring. It was the sleepiest of +sleepy days. The drowsing murmur of the five and twenty studying +scholars soothed the soul like the spell that is in the murmur of bees. +Away off in the flaming sunshine, Cardiff Hill lifted its soft green +sides through a shimmering veil of heat, tinted with the purple of +distance; a few birds floated on lazy wing high in the air; no other +living thing was visible but some cows, and they were asleep. Tom's +heart ached to be free, or else to have something of interest to do to +pass the dreary time. His hand wandered into his pocket and his face +lit up with a glow of gratitude that was prayer, though he did not know +it. Then furtively the percussion-cap box came out. He released the +tick and put him on the long flat desk. The creature probably glowed +with a gratitude that amounted to prayer, too, at this moment, but it +was premature: for when he started thankfully to travel off, Tom turned +him aside with a pin and made him take a new direction. + +Tom's bosom friend sat next him, suffering just as Tom had been, and +now he was deeply and gratefully interested in this entertainment in an +instant. This bosom friend was Joe Harper. The two boys were sworn +friends all the week, and embattled enemies on Saturdays. Joe took a +pin out of his lapel and began to assist in exercising the prisoner. +The sport grew in interest momently. Soon Tom said that they were +interfering with each other, and neither getting the fullest benefit of +the tick. So he put Joe's slate on the desk and drew a line down the +middle of it from top to bottom. + +"Now," said he, "as long as he is on your side you can stir him up and +I'll let him alone; but if you let him get away and get on my side, +you're to leave him alone as long as I can keep him from crossing over." + +"All right, go ahead; start him up." + +The tick escaped from Tom, presently, and crossed the equator. Joe +harassed him awhile, and then he got away and crossed back again. This +change of base occurred often. While one boy was worrying the tick with +absorbing interest, the other would look on with interest as strong, +the two heads bowed together over the slate, and the two souls dead to +all things else. At last luck seemed to settle and abide with Joe. The +tick tried this, that, and the other course, and got as excited and as +anxious as the boys themselves, but time and again just as he would +have victory in his very grasp, so to speak, and Tom's fingers would be +twitching to begin, Joe's pin would deftly head him off, and keep +possession. At last Tom could stand it no longer. The temptation was +too strong. So he reached out and lent a hand with his pin. Joe was +angry in a moment. Said he: + +"Tom, you let him alone." + +"I only just want to stir him up a little, Joe." + +"No, sir, it ain't fair; you just let him alone." + +"Blame it, I ain't going to stir him much." + +"Let him alone, I tell you." + +"I won't!" + +"You shall--he's on my side of the line." + +"Look here, Joe Harper, whose is that tick?" + +"I don't care whose tick he is--he's on my side of the line, and you +sha'n't touch him." + +"Well, I'll just bet I will, though. He's my tick and I'll do what I +blame please with him, or die!" + +A tremendous whack came down on Tom's shoulders, and its duplicate on +Joe's; and for the space of two minutes the dust continued to fly from +the two jackets and the whole school to enjoy it. The boys had been too +absorbed to notice the hush that had stolen upon the school awhile +before when the master came tiptoeing down the room and stood over +them. He had contemplated a good part of the performance before he +contributed his bit of variety to it. + +When school broke up at noon, Tom flew to Becky Thatcher, and +whispered in her ear: + +"Put on your bonnet and let on you're going home; and when you get to +the corner, give the rest of 'em the slip, and turn down through the +lane and come back. I'll go the other way and come it over 'em the same +way." + +So the one went off with one group of scholars, and the other with +another. In a little while the two met at the bottom of the lane, and +when they reached the school they had it all to themselves. Then they +sat together, with a slate before them, and Tom gave Becky the pencil +and held her hand in his, guiding it, and so created another surprising +house. When the interest in art began to wane, the two fell to talking. +Tom was swimming in bliss. He said: + +"Do you love rats?" + +"No! I hate them!" + +"Well, I do, too--LIVE ones. But I mean dead ones, to swing round your +head with a string." + +"No, I don't care for rats much, anyway. What I like is chewing-gum." + +"Oh, I should say so! I wish I had some now." + +"Do you? I've got some. I'll let you chew it awhile, but you must give +it back to me." + +That was agreeable, so they chewed it turn about, and dangled their +legs against the bench in excess of contentment. + +"Was you ever at a circus?" said Tom. + +"Yes, and my pa's going to take me again some time, if I'm good." + +"I been to the circus three or four times--lots of times. Church ain't +shucks to a circus. There's things going on at a circus all the time. +I'm going to be a clown in a circus when I grow up." + +"Oh, are you! That will be nice. They're so lovely, all spotted up." + +"Yes, that's so. And they get slathers of money--most a dollar a day, +Ben Rogers says. Say, Becky, was you ever engaged?" + +"What's that?" + +"Why, engaged to be married." + +"No." + +"Would you like to?" + +"I reckon so. I don't know. What is it like?" + +"Like? Why it ain't like anything. You only just tell a boy you won't +ever have anybody but him, ever ever ever, and then you kiss and that's +all. Anybody can do it." + +"Kiss? What do you kiss for?" + +"Why, that, you know, is to--well, they always do that." + +"Everybody?" + +"Why, yes, everybody that's in love with each other. Do you remember +what I wrote on the slate?" + +"Ye--yes." + +"What was it?" + +"I sha'n't tell you." + +"Shall I tell YOU?" + +"Ye--yes--but some other time." + +"No, now." + +"No, not now--to-morrow." + +"Oh, no, NOW. Please, Becky--I'll whisper it, I'll whisper it ever so +easy." + +Becky hesitating, Tom took silence for consent, and passed his arm +about her waist and whispered the tale ever so softly, with his mouth +close to her ear. And then he added: + +"Now you whisper it to me--just the same." + +She resisted, for a while, and then said: + +"You turn your face away so you can't see, and then I will. But you +mustn't ever tell anybody--WILL you, Tom? Now you won't, WILL you?" + +"No, indeed, indeed I won't. Now, Becky." + +He turned his face away. She bent timidly around till her breath +stirred his curls and whispered, "I--love--you!" + +Then she sprang away and ran around and around the desks and benches, +with Tom after her, and took refuge in a corner at last, with her +little white apron to her face. Tom clasped her about her neck and +pleaded: + +"Now, Becky, it's all done--all over but the kiss. Don't you be afraid +of that--it ain't anything at all. Please, Becky." And he tugged at her +apron and the hands. + +By and by she gave up, and let her hands drop; her face, all glowing +with the struggle, came up and submitted. Tom kissed the red lips and +said: + +"Now it's all done, Becky. And always after this, you know, you ain't +ever to love anybody but me, and you ain't ever to marry anybody but +me, ever never and forever. Will you?" + +"No, I'll never love anybody but you, Tom, and I'll never marry +anybody but you--and you ain't to ever marry anybody but me, either." + +"Certainly. Of course. That's PART of it. And always coming to school +or when we're going home, you're to walk with me, when there ain't +anybody looking--and you choose me and I choose you at parties, because +that's the way you do when you're engaged." + +"It's so nice. I never heard of it before." + +"Oh, it's ever so gay! Why, me and Amy Lawrence--" + +The big eyes told Tom his blunder and he stopped, confused. + +"Oh, Tom! Then I ain't the first you've ever been engaged to!" + +The child began to cry. Tom said: + +"Oh, don't cry, Becky, I don't care for her any more." + +"Yes, you do, Tom--you know you do." + +Tom tried to put his arm about her neck, but she pushed him away and +turned her face to the wall, and went on crying. Tom tried again, with +soothing words in his mouth, and was repulsed again. Then his pride was +up, and he strode away and went outside. He stood about, restless and +uneasy, for a while, glancing at the door, every now and then, hoping +she would repent and come to find him. But she did not. Then he began +to feel badly and fear that he was in the wrong. It was a hard struggle +with him to make new advances, now, but he nerved himself to it and +entered. She was still standing back there in the corner, sobbing, with +her face to the wall. Tom's heart smote him. He went to her and stood a +moment, not knowing exactly how to proceed. Then he said hesitatingly: + +"Becky, I--I don't care for anybody but you." + +No reply--but sobs. + +"Becky"--pleadingly. "Becky, won't you say something?" + +More sobs. + +Tom got out his chiefest jewel, a brass knob from the top of an +andiron, and passed it around her so that she could see it, and said: + +"Please, Becky, won't you take it?" + +She struck it to the floor. Then Tom marched out of the house and over +the hills and far away, to return to school no more that day. Presently +Becky began to suspect. She ran to the door; he was not in sight; she +flew around to the play-yard; he was not there. Then she called: + +"Tom! Come back, Tom!" + +She listened intently, but there was no answer. She had no companions +but silence and loneliness. So she sat down to cry again and upbraid +herself; and by this time the scholars began to gather again, and she +had to hide her griefs and still her broken heart and take up the cross +of a long, dreary, aching afternoon, with none among the strangers +about her to exchange sorrows with. + + + +CHAPTER VIII + +TOM dodged hither and thither through lanes until he was well out of +the track of returning scholars, and then fell into a moody jog. He +crossed a small "branch" two or three times, because of a prevailing +juvenile superstition that to cross water baffled pursuit. Half an hour +later he was disappearing behind the Douglas mansion on the summit of +Cardiff Hill, and the schoolhouse was hardly distinguishable away off +in the valley behind him. He entered a dense wood, picked his pathless +way to the centre of it, and sat down on a mossy spot under a spreading +oak. There was not even a zephyr stirring; the dead noonday heat had +even stilled the songs of the birds; nature lay in a trance that was +broken by no sound but the occasional far-off hammering of a +woodpecker, and this seemed to render the pervading silence and sense +of loneliness the more profound. The boy's soul was steeped in +melancholy; his feelings were in happy accord with his surroundings. He +sat long with his elbows on his knees and his chin in his hands, +meditating. It seemed to him that life was but a trouble, at best, and +he more than half envied Jimmy Hodges, so lately released; it must be +very peaceful, he thought, to lie and slumber and dream forever and +ever, with the wind whispering through the trees and caressing the +grass and the flowers over the grave, and nothing to bother and grieve +about, ever any more. If he only had a clean Sunday-school record he +could be willing to go, and be done with it all. Now as to this girl. +What had he done? Nothing. He had meant the best in the world, and been +treated like a dog--like a very dog. She would be sorry some day--maybe +when it was too late. Ah, if he could only die TEMPORARILY! + +But the elastic heart of youth cannot be compressed into one +constrained shape long at a time. Tom presently began to drift +insensibly back into the concerns of this life again. What if he turned +his back, now, and disappeared mysteriously? What if he went away--ever +so far away, into unknown countries beyond the seas--and never came +back any more! How would she feel then! The idea of being a clown +recurred to him now, only to fill him with disgust. For frivolity and +jokes and spotted tights were an offense, when they intruded themselves +upon a spirit that was exalted into the vague august realm of the +romantic. No, he would be a soldier, and return after long years, all +war-worn and illustrious. No--better still, he would join the Indians, +and hunt buffaloes and go on the warpath in the mountain ranges and the +trackless great plains of the Far West, and away in the future come +back a great chief, bristling with feathers, hideous with paint, and +prance into Sunday-school, some drowsy summer morning, with a +bloodcurdling war-whoop, and sear the eyeballs of all his companions +with unappeasable envy. But no, there was something gaudier even than +this. He would be a pirate! That was it! NOW his future lay plain +before him, and glowing with unimaginable splendor. How his name would +fill the world, and make people shudder! How gloriously he would go +plowing the dancing seas, in his long, low, black-hulled racer, the +Spirit of the Storm, with his grisly flag flying at the fore! And at +the zenith of his fame, how he would suddenly appear at the old village +and stalk into church, brown and weather-beaten, in his black velvet +doublet and trunks, his great jack-boots, his crimson sash, his belt +bristling with horse-pistols, his crime-rusted cutlass at his side, his +slouch hat with waving plumes, his black flag unfurled, with the skull +and crossbones on it, and hear with swelling ecstasy the whisperings, +"It's Tom Sawyer the Pirate!--the Black Avenger of the Spanish Main!" + +Yes, it was settled; his career was determined. He would run away from +home and enter upon it. He would start the very next morning. Therefore +he must now begin to get ready. He would collect his resources +together. He went to a rotten log near at hand and began to dig under +one end of it with his Barlow knife. He soon struck wood that sounded +hollow. He put his hand there and uttered this incantation impressively: + +"What hasn't come here, come! What's here, stay here!" + +Then he scraped away the dirt, and exposed a pine shingle. He took it +up and disclosed a shapely little treasure-house whose bottom and sides +were of shingles. In it lay a marble. Tom's astonishment was boundless! +He scratched his head with a perplexed air, and said: + +"Well, that beats anything!" + +Then he tossed the marble away pettishly, and stood cogitating. The +truth was, that a superstition of his had failed, here, which he and +all his comrades had always looked upon as infallible. If you buried a +marble with certain necessary incantations, and left it alone a +fortnight, and then opened the place with the incantation he had just +used, you would find that all the marbles you had ever lost had +gathered themselves together there, meantime, no matter how widely they +had been separated. But now, this thing had actually and unquestionably +failed. Tom's whole structure of faith was shaken to its foundations. +He had many a time heard of this thing succeeding but never of its +failing before. It did not occur to him that he had tried it several +times before, himself, but could never find the hiding-places +afterward. He puzzled over the matter some time, and finally decided +that some witch had interfered and broken the charm. He thought he +would satisfy himself on that point; so he searched around till he +found a small sandy spot with a little funnel-shaped depression in it. +He laid himself down and put his mouth close to this depression and +called-- + +"Doodle-bug, doodle-bug, tell me what I want to know! Doodle-bug, +doodle-bug, tell me what I want to know!" + +The sand began to work, and presently a small black bug appeared for a +second and then darted under again in a fright. + +"He dasn't tell! So it WAS a witch that done it. I just knowed it." + +He well knew the futility of trying to contend against witches, so he +gave up discouraged. But it occurred to him that he might as well have +the marble he had just thrown away, and therefore he went and made a +patient search for it. But he could not find it. Now he went back to +his treasure-house and carefully placed himself just as he had been +standing when he tossed the marble away; then he took another marble +from his pocket and tossed it in the same way, saying: + +"Brother, go find your brother!" + +He watched where it stopped, and went there and looked. But it must +have fallen short or gone too far; so he tried twice more. The last +repetition was successful. The two marbles lay within a foot of each +other. + +Just here the blast of a toy tin trumpet came faintly down the green +aisles of the forest. Tom flung off his jacket and trousers, turned a +suspender into a belt, raked away some brush behind the rotten log, +disclosing a rude bow and arrow, a lath sword and a tin trumpet, and in +a moment had seized these things and bounded away, barelegged, with +fluttering shirt. He presently halted under a great elm, blew an +answering blast, and then began to tiptoe and look warily out, this way +and that. He said cautiously--to an imaginary company: + +"Hold, my merry men! Keep hid till I blow." + +Now appeared Joe Harper, as airily clad and elaborately armed as Tom. +Tom called: + +"Hold! Who comes here into Sherwood Forest without my pass?" + +"Guy of Guisborne wants no man's pass. Who art thou that--that--" + +"Dares to hold such language," said Tom, prompting--for they talked +"by the book," from memory. + +"Who art thou that dares to hold such language?" + +"I, indeed! I am Robin Hood, as thy caitiff carcase soon shall know." + +"Then art thou indeed that famous outlaw? Right gladly will I dispute +with thee the passes of the merry wood. Have at thee!" + +They took their lath swords, dumped their other traps on the ground, +struck a fencing attitude, foot to foot, and began a grave, careful +combat, "two up and two down." Presently Tom said: + +"Now, if you've got the hang, go it lively!" + +So they "went it lively," panting and perspiring with the work. By and +by Tom shouted: + +"Fall! fall! Why don't you fall?" + +"I sha'n't! Why don't you fall yourself? You're getting the worst of +it." + +"Why, that ain't anything. I can't fall; that ain't the way it is in +the book. The book says, 'Then with one back-handed stroke he slew poor +Guy of Guisborne.' You're to turn around and let me hit you in the +back." + +There was no getting around the authorities, so Joe turned, received +the whack and fell. + +"Now," said Joe, getting up, "you got to let me kill YOU. That's fair." + +"Why, I can't do that, it ain't in the book." + +"Well, it's blamed mean--that's all." + +"Well, say, Joe, you can be Friar Tuck or Much the miller's son, and +lam me with a quarter-staff; or I'll be the Sheriff of Nottingham and +you be Robin Hood a little while and kill me." + +This was satisfactory, and so these adventures were carried out. Then +Tom became Robin Hood again, and was allowed by the treacherous nun to +bleed his strength away through his neglected wound. And at last Joe, +representing a whole tribe of weeping outlaws, dragged him sadly forth, +gave his bow into his feeble hands, and Tom said, "Where this arrow +falls, there bury poor Robin Hood under the greenwood tree." Then he +shot the arrow and fell back and would have died, but he lit on a +nettle and sprang up too gaily for a corpse. + +The boys dressed themselves, hid their accoutrements, and went off +grieving that there were no outlaws any more, and wondering what modern +civilization could claim to have done to compensate for their loss. +They said they would rather be outlaws a year in Sherwood Forest than +President of the United States forever. + + + +CHAPTER IX + +AT half-past nine, that night, Tom and Sid were sent to bed, as usual. +They said their prayers, and Sid was soon asleep. Tom lay awake and +waited, in restless impatience. When it seemed to him that it must be +nearly daylight, he heard the clock strike ten! This was despair. He +would have tossed and fidgeted, as his nerves demanded, but he was +afraid he might wake Sid. So he lay still, and stared up into the dark. +Everything was dismally still. By and by, out of the stillness, little, +scarcely perceptible noises began to emphasize themselves. The ticking +of the clock began to bring itself into notice. Old beams began to +crack mysteriously. The stairs creaked faintly. Evidently spirits were +abroad. A measured, muffled snore issued from Aunt Polly's chamber. And +now the tiresome chirping of a cricket that no human ingenuity could +locate, began. Next the ghastly ticking of a deathwatch in the wall at +the bed's head made Tom shudder--it meant that somebody's days were +numbered. Then the howl of a far-off dog rose on the night air, and was +answered by a fainter howl from a remoter distance. Tom was in an +agony. At last he was satisfied that time had ceased and eternity +begun; he began to doze, in spite of himself; the clock chimed eleven, +but he did not hear it. And then there came, mingling with his +half-formed dreams, a most melancholy caterwauling. The raising of a +neighboring window disturbed him. A cry of "Scat! you devil!" and the +crash of an empty bottle against the back of his aunt's woodshed +brought him wide awake, and a single minute later he was dressed and +out of the window and creeping along the roof of the "ell" on all +fours. He "meow'd" with caution once or twice, as he went; then jumped +to the roof of the woodshed and thence to the ground. Huckleberry Finn +was there, with his dead cat. The boys moved off and disappeared in the +gloom. At the end of half an hour they were wading through the tall +grass of the graveyard. + +It was a graveyard of the old-fashioned Western kind. It was on a +hill, about a mile and a half from the village. It had a crazy board +fence around it, which leaned inward in places, and outward the rest of +the time, but stood upright nowhere. Grass and weeds grew rank over the +whole cemetery. All the old graves were sunken in, there was not a +tombstone on the place; round-topped, worm-eaten boards staggered over +the graves, leaning for support and finding none. "Sacred to the memory +of" So-and-So had been painted on them once, but it could no longer +have been read, on the most of them, now, even if there had been light. + +A faint wind moaned through the trees, and Tom feared it might be the +spirits of the dead, complaining at being disturbed. The boys talked +little, and only under their breath, for the time and the place and the +pervading solemnity and silence oppressed their spirits. They found the +sharp new heap they were seeking, and ensconced themselves within the +protection of three great elms that grew in a bunch within a few feet +of the grave. + +Then they waited in silence for what seemed a long time. The hooting +of a distant owl was all the sound that troubled the dead stillness. +Tom's reflections grew oppressive. He must force some talk. So he said +in a whisper: + +"Hucky, do you believe the dead people like it for us to be here?" + +Huckleberry whispered: + +"I wisht I knowed. It's awful solemn like, AIN'T it?" + +"I bet it is." + +There was a considerable pause, while the boys canvassed this matter +inwardly. Then Tom whispered: + +"Say, Hucky--do you reckon Hoss Williams hears us talking?" + +"O' course he does. Least his sperrit does." + +Tom, after a pause: + +"I wish I'd said Mister Williams. But I never meant any harm. +Everybody calls him Hoss." + +"A body can't be too partic'lar how they talk 'bout these-yer dead +people, Tom." + +This was a damper, and conversation died again. + +Presently Tom seized his comrade's arm and said: + +"Sh!" + +"What is it, Tom?" And the two clung together with beating hearts. + +"Sh! There 'tis again! Didn't you hear it?" + +"I--" + +"There! Now you hear it." + +"Lord, Tom, they're coming! They're coming, sure. What'll we do?" + +"I dono. Think they'll see us?" + +"Oh, Tom, they can see in the dark, same as cats. I wisht I hadn't +come." + +"Oh, don't be afeard. I don't believe they'll bother us. We ain't +doing any harm. If we keep perfectly still, maybe they won't notice us +at all." + +"I'll try to, Tom, but, Lord, I'm all of a shiver." + +"Listen!" + +The boys bent their heads together and scarcely breathed. A muffled +sound of voices floated up from the far end of the graveyard. + +"Look! See there!" whispered Tom. "What is it?" + +"It's devil-fire. Oh, Tom, this is awful." + +Some vague figures approached through the gloom, swinging an +old-fashioned tin lantern that freckled the ground with innumerable +little spangles of light. Presently Huckleberry whispered with a +shudder: + +"It's the devils sure enough. Three of 'em! Lordy, Tom, we're goners! +Can you pray?" + +"I'll try, but don't you be afeard. They ain't going to hurt us. 'Now +I lay me down to sleep, I--'" + +"Sh!" + +"What is it, Huck?" + +"They're HUMANS! One of 'em is, anyway. One of 'em's old Muff Potter's +voice." + +"No--'tain't so, is it?" + +"I bet I know it. Don't you stir nor budge. He ain't sharp enough to +notice us. Drunk, the same as usual, likely--blamed old rip!" + +"All right, I'll keep still. Now they're stuck. Can't find it. Here +they come again. Now they're hot. Cold again. Hot again. Red hot! +They're p'inted right, this time. Say, Huck, I know another o' them +voices; it's Injun Joe." + +"That's so--that murderin' half-breed! I'd druther they was devils a +dern sight. What kin they be up to?" + +The whisper died wholly out, now, for the three men had reached the +grave and stood within a few feet of the boys' hiding-place. + +"Here it is," said the third voice; and the owner of it held the +lantern up and revealed the face of young Doctor Robinson. + +Potter and Injun Joe were carrying a handbarrow with a rope and a +couple of shovels on it. They cast down their load and began to open +the grave. The doctor put the lantern at the head of the grave and came +and sat down with his back against one of the elm trees. He was so +close the boys could have touched him. + +"Hurry, men!" he said, in a low voice; "the moon might come out at any +moment." + +They growled a response and went on digging. For some time there was +no noise but the grating sound of the spades discharging their freight +of mould and gravel. It was very monotonous. Finally a spade struck +upon the coffin with a dull woody accent, and within another minute or +two the men had hoisted it out on the ground. They pried off the lid +with their shovels, got out the body and dumped it rudely on the +ground. The moon drifted from behind the clouds and exposed the pallid +face. The barrow was got ready and the corpse placed on it, covered +with a blanket, and bound to its place with the rope. Potter took out a +large spring-knife and cut off the dangling end of the rope and then +said: + +"Now the cussed thing's ready, Sawbones, and you'll just out with +another five, or here she stays." + +"That's the talk!" said Injun Joe. + +"Look here, what does this mean?" said the doctor. "You required your +pay in advance, and I've paid you." + +"Yes, and you done more than that," said Injun Joe, approaching the +doctor, who was now standing. "Five years ago you drove me away from +your father's kitchen one night, when I come to ask for something to +eat, and you said I warn't there for any good; and when I swore I'd get +even with you if it took a hundred years, your father had me jailed for +a vagrant. Did you think I'd forget? The Injun blood ain't in me for +nothing. And now I've GOT you, and you got to SETTLE, you know!" + +He was threatening the doctor, with his fist in his face, by this +time. The doctor struck out suddenly and stretched the ruffian on the +ground. Potter dropped his knife, and exclaimed: + +"Here, now, don't you hit my pard!" and the next moment he had +grappled with the doctor and the two were struggling with might and +main, trampling the grass and tearing the ground with their heels. +Injun Joe sprang to his feet, his eyes flaming with passion, snatched +up Potter's knife, and went creeping, catlike and stooping, round and +round about the combatants, seeking an opportunity. All at once the +doctor flung himself free, seized the heavy headboard of Williams' +grave and felled Potter to the earth with it--and in the same instant +the half-breed saw his chance and drove the knife to the hilt in the +young man's breast. He reeled and fell partly upon Potter, flooding him +with his blood, and in the same moment the clouds blotted out the +dreadful spectacle and the two frightened boys went speeding away in +the dark. + +Presently, when the moon emerged again, Injun Joe was standing over +the two forms, contemplating them. The doctor murmured inarticulately, +gave a long gasp or two and was still. The half-breed muttered: + +"THAT score is settled--damn you." + +Then he robbed the body. After which he put the fatal knife in +Potter's open right hand, and sat down on the dismantled coffin. Three +--four--five minutes passed, and then Potter began to stir and moan. His +hand closed upon the knife; he raised it, glanced at it, and let it +fall, with a shudder. Then he sat up, pushing the body from him, and +gazed at it, and then around him, confusedly. His eyes met Joe's. + +"Lord, how is this, Joe?" he said. + +"It's a dirty business," said Joe, without moving. + +"What did you do it for?" + +"I! I never done it!" + +"Look here! That kind of talk won't wash." + +Potter trembled and grew white. + +"I thought I'd got sober. I'd no business to drink to-night. But it's +in my head yet--worse'n when we started here. I'm all in a muddle; +can't recollect anything of it, hardly. Tell me, Joe--HONEST, now, old +feller--did I do it? Joe, I never meant to--'pon my soul and honor, I +never meant to, Joe. Tell me how it was, Joe. Oh, it's awful--and him +so young and promising." + +"Why, you two was scuffling, and he fetched you one with the headboard +and you fell flat; and then up you come, all reeling and staggering +like, and snatched the knife and jammed it into him, just as he fetched +you another awful clip--and here you've laid, as dead as a wedge til +now." + +"Oh, I didn't know what I was a-doing. I wish I may die this minute if +I did. It was all on account of the whiskey and the excitement, I +reckon. I never used a weepon in my life before, Joe. I've fought, but +never with weepons. They'll all say that. Joe, don't tell! Say you +won't tell, Joe--that's a good feller. I always liked you, Joe, and +stood up for you, too. Don't you remember? You WON'T tell, WILL you, +Joe?" And the poor creature dropped on his knees before the stolid +murderer, and clasped his appealing hands. + +"No, you've always been fair and square with me, Muff Potter, and I +won't go back on you. There, now, that's as fair as a man can say." + +"Oh, Joe, you're an angel. I'll bless you for this the longest day I +live." And Potter began to cry. + +"Come, now, that's enough of that. This ain't any time for blubbering. +You be off yonder way and I'll go this. Move, now, and don't leave any +tracks behind you." + +Potter started on a trot that quickly increased to a run. The +half-breed stood looking after him. He muttered: + +"If he's as much stunned with the lick and fuddled with the rum as he +had the look of being, he won't think of the knife till he's gone so +far he'll be afraid to come back after it to such a place by himself +--chicken-heart!" + +Two or three minutes later the murdered man, the blanketed corpse, the +lidless coffin, and the open grave were under no inspection but the +moon's. The stillness was complete again, too. + + + +CHAPTER X + +THE two boys flew on and on, toward the village, speechless with +horror. They glanced backward over their shoulders from time to time, +apprehensively, as if they feared they might be followed. Every stump +that started up in their path seemed a man and an enemy, and made them +catch their breath; and as they sped by some outlying cottages that lay +near the village, the barking of the aroused watch-dogs seemed to give +wings to their feet. + +"If we can only get to the old tannery before we break down!" +whispered Tom, in short catches between breaths. "I can't stand it much +longer." + +Huckleberry's hard pantings were his only reply, and the boys fixed +their eyes on the goal of their hopes and bent to their work to win it. +They gained steadily on it, and at last, breast to breast, they burst +through the open door and fell grateful and exhausted in the sheltering +shadows beyond. By and by their pulses slowed down, and Tom whispered: + +"Huckleberry, what do you reckon'll come of this?" + +"If Doctor Robinson dies, I reckon hanging'll come of it." + +"Do you though?" + +"Why, I KNOW it, Tom." + +Tom thought a while, then he said: + +"Who'll tell? We?" + +"What are you talking about? S'pose something happened and Injun Joe +DIDN'T hang? Why, he'd kill us some time or other, just as dead sure as +we're a laying here." + +"That's just what I was thinking to myself, Huck." + +"If anybody tells, let Muff Potter do it, if he's fool enough. He's +generally drunk enough." + +Tom said nothing--went on thinking. Presently he whispered: + +"Huck, Muff Potter don't know it. How can he tell?" + +"What's the reason he don't know it?" + +"Because he'd just got that whack when Injun Joe done it. D'you reckon +he could see anything? D'you reckon he knowed anything?" + +"By hokey, that's so, Tom!" + +"And besides, look-a-here--maybe that whack done for HIM!" + +"No, 'taint likely, Tom. He had liquor in him; I could see that; and +besides, he always has. Well, when pap's full, you might take and belt +him over the head with a church and you couldn't phase him. He says so, +his own self. So it's the same with Muff Potter, of course. But if a +man was dead sober, I reckon maybe that whack might fetch him; I dono." + +After another reflective silence, Tom said: + +"Hucky, you sure you can keep mum?" + +"Tom, we GOT to keep mum. You know that. That Injun devil wouldn't +make any more of drownding us than a couple of cats, if we was to +squeak 'bout this and they didn't hang him. Now, look-a-here, Tom, less +take and swear to one another--that's what we got to do--swear to keep +mum." + +"I'm agreed. It's the best thing. Would you just hold hands and swear +that we--" + +"Oh no, that wouldn't do for this. That's good enough for little +rubbishy common things--specially with gals, cuz THEY go back on you +anyway, and blab if they get in a huff--but there orter be writing +'bout a big thing like this. And blood." + +Tom's whole being applauded this idea. It was deep, and dark, and +awful; the hour, the circumstances, the surroundings, were in keeping +with it. He picked up a clean pine shingle that lay in the moonlight, +took a little fragment of "red keel" out of his pocket, got the moon on +his work, and painfully scrawled these lines, emphasizing each slow +down-stroke by clamping his tongue between his teeth, and letting up +the pressure on the up-strokes. [See next page.] + + "Huck Finn and + Tom Sawyer swears + they will keep mum + about This and They + wish They may Drop + down dead in Their + Tracks if They ever + Tell and Rot." + +Huckleberry was filled with admiration of Tom's facility in writing, +and the sublimity of his language. He at once took a pin from his lapel +and was going to prick his flesh, but Tom said: + +"Hold on! Don't do that. A pin's brass. It might have verdigrease on +it." + +"What's verdigrease?" + +"It's p'ison. That's what it is. You just swaller some of it once +--you'll see." + +So Tom unwound the thread from one of his needles, and each boy +pricked the ball of his thumb and squeezed out a drop of blood. In +time, after many squeezes, Tom managed to sign his initials, using the +ball of his little finger for a pen. Then he showed Huckleberry how to +make an H and an F, and the oath was complete. They buried the shingle +close to the wall, with some dismal ceremonies and incantations, and +the fetters that bound their tongues were considered to be locked and +the key thrown away. + +A figure crept stealthily through a break in the other end of the +ruined building, now, but they did not notice it. + +"Tom," whispered Huckleberry, "does this keep us from EVER telling +--ALWAYS?" + +"Of course it does. It don't make any difference WHAT happens, we got +to keep mum. We'd drop down dead--don't YOU know that?" + +"Yes, I reckon that's so." + +They continued to whisper for some little time. Presently a dog set up +a long, lugubrious howl just outside--within ten feet of them. The boys +clasped each other suddenly, in an agony of fright. + +"Which of us does he mean?" gasped Huckleberry. + +"I dono--peep through the crack. Quick!" + +"No, YOU, Tom!" + +"I can't--I can't DO it, Huck!" + +"Please, Tom. There 'tis again!" + +"Oh, lordy, I'm thankful!" whispered Tom. "I know his voice. It's Bull +Harbison." * + +[* If Mr. Harbison owned a slave named Bull, Tom would have spoken of +him as "Harbison's Bull," but a son or a dog of that name was "Bull +Harbison."] + +"Oh, that's good--I tell you, Tom, I was most scared to death; I'd a +bet anything it was a STRAY dog." + +The dog howled again. The boys' hearts sank once more. + +"Oh, my! that ain't no Bull Harbison!" whispered Huckleberry. "DO, Tom!" + +Tom, quaking with fear, yielded, and put his eye to the crack. His +whisper was hardly audible when he said: + +"Oh, Huck, IT S A STRAY DOG!" + +"Quick, Tom, quick! Who does he mean?" + +"Huck, he must mean us both--we're right together." + +"Oh, Tom, I reckon we're goners. I reckon there ain't no mistake 'bout +where I'LL go to. I been so wicked." + +"Dad fetch it! This comes of playing hookey and doing everything a +feller's told NOT to do. I might a been good, like Sid, if I'd a tried +--but no, I wouldn't, of course. But if ever I get off this time, I lay +I'll just WALLER in Sunday-schools!" And Tom began to snuffle a little. + +"YOU bad!" and Huckleberry began to snuffle too. "Consound it, Tom +Sawyer, you're just old pie, 'longside o' what I am. Oh, LORDY, lordy, +lordy, I wisht I only had half your chance." + +Tom choked off and whispered: + +"Look, Hucky, look! He's got his BACK to us!" + +Hucky looked, with joy in his heart. + +"Well, he has, by jingoes! Did he before?" + +"Yes, he did. But I, like a fool, never thought. Oh, this is bully, +you know. NOW who can he mean?" + +The howling stopped. Tom pricked up his ears. + +"Sh! What's that?" he whispered. + +"Sounds like--like hogs grunting. No--it's somebody snoring, Tom." + +"That IS it! Where 'bouts is it, Huck?" + +"I bleeve it's down at 'tother end. Sounds so, anyway. Pap used to +sleep there, sometimes, 'long with the hogs, but laws bless you, he +just lifts things when HE snores. Besides, I reckon he ain't ever +coming back to this town any more." + +The spirit of adventure rose in the boys' souls once more. + +"Hucky, do you das't to go if I lead?" + +"I don't like to, much. Tom, s'pose it's Injun Joe!" + +Tom quailed. But presently the temptation rose up strong again and the +boys agreed to try, with the understanding that they would take to +their heels if the snoring stopped. So they went tiptoeing stealthily +down, the one behind the other. When they had got to within five steps +of the snorer, Tom stepped on a stick, and it broke with a sharp snap. +The man moaned, writhed a little, and his face came into the moonlight. +It was Muff Potter. The boys' hearts had stood still, and their hopes +too, when the man moved, but their fears passed away now. They tiptoed +out, through the broken weather-boarding, and stopped at a little +distance to exchange a parting word. That long, lugubrious howl rose on +the night air again! They turned and saw the strange dog standing +within a few feet of where Potter was lying, and FACING Potter, with +his nose pointing heavenward. + +"Oh, geeminy, it's HIM!" exclaimed both boys, in a breath. + +"Say, Tom--they say a stray dog come howling around Johnny Miller's +house, 'bout midnight, as much as two weeks ago; and a whippoorwill +come in and lit on the banisters and sung, the very same evening; and +there ain't anybody dead there yet." + +"Well, I know that. And suppose there ain't. Didn't Gracie Miller fall +in the kitchen fire and burn herself terrible the very next Saturday?" + +"Yes, but she ain't DEAD. And what's more, she's getting better, too." + +"All right, you wait and see. She's a goner, just as dead sure as Muff +Potter's a goner. That's what the niggers say, and they know all about +these kind of things, Huck." + +Then they separated, cogitating. When Tom crept in at his bedroom +window the night was almost spent. He undressed with excessive caution, +and fell asleep congratulating himself that nobody knew of his +escapade. He was not aware that the gently-snoring Sid was awake, and +had been so for an hour. + +When Tom awoke, Sid was dressed and gone. There was a late look in the +light, a late sense in the atmosphere. He was startled. Why had he not +been called--persecuted till he was up, as usual? The thought filled +him with bodings. Within five minutes he was dressed and down-stairs, +feeling sore and drowsy. The family were still at table, but they had +finished breakfast. There was no voice of rebuke; but there were +averted eyes; there was a silence and an air of solemnity that struck a +chill to the culprit's heart. He sat down and tried to seem gay, but it +was up-hill work; it roused no smile, no response, and he lapsed into +silence and let his heart sink down to the depths. + +After breakfast his aunt took him aside, and Tom almost brightened in +the hope that he was going to be flogged; but it was not so. His aunt +wept over him and asked him how he could go and break her old heart so; +and finally told him to go on, and ruin himself and bring her gray +hairs with sorrow to the grave, for it was no use for her to try any +more. This was worse than a thousand whippings, and Tom's heart was +sorer now than his body. He cried, he pleaded for forgiveness, promised +to reform over and over again, and then received his dismissal, feeling +that he had won but an imperfect forgiveness and established but a +feeble confidence. + +He left the presence too miserable to even feel revengeful toward Sid; +and so the latter's prompt retreat through the back gate was +unnecessary. He moped to school gloomy and sad, and took his flogging, +along with Joe Harper, for playing hookey the day before, with the air +of one whose heart was busy with heavier woes and wholly dead to +trifles. Then he betook himself to his seat, rested his elbows on his +desk and his jaws in his hands, and stared at the wall with the stony +stare of suffering that has reached the limit and can no further go. +His elbow was pressing against some hard substance. After a long time +he slowly and sadly changed his position, and took up this object with +a sigh. It was in a paper. He unrolled it. A long, lingering, colossal +sigh followed, and his heart broke. It was his brass andiron knob! + +This final feather broke the camel's back. + + + +CHAPTER XI + +CLOSE upon the hour of noon the whole village was suddenly electrified +with the ghastly news. No need of the as yet undreamed-of telegraph; +the tale flew from man to man, from group to group, from house to +house, with little less than telegraphic speed. Of course the +schoolmaster gave holiday for that afternoon; the town would have +thought strangely of him if he had not. + +A gory knife had been found close to the murdered man, and it had been +recognized by somebody as belonging to Muff Potter--so the story ran. +And it was said that a belated citizen had come upon Potter washing +himself in the "branch" about one or two o'clock in the morning, and +that Potter had at once sneaked off--suspicious circumstances, +especially the washing which was not a habit with Potter. It was also +said that the town had been ransacked for this "murderer" (the public +are not slow in the matter of sifting evidence and arriving at a +verdict), but that he could not be found. Horsemen had departed down +all the roads in every direction, and the Sheriff "was confident" that +he would be captured before night. + +All the town was drifting toward the graveyard. Tom's heartbreak +vanished and he joined the procession, not because he would not a +thousand times rather go anywhere else, but because an awful, +unaccountable fascination drew him on. Arrived at the dreadful place, +he wormed his small body through the crowd and saw the dismal +spectacle. It seemed to him an age since he was there before. Somebody +pinched his arm. He turned, and his eyes met Huckleberry's. Then both +looked elsewhere at once, and wondered if anybody had noticed anything +in their mutual glance. But everybody was talking, and intent upon the +grisly spectacle before them. + +"Poor fellow!" "Poor young fellow!" "This ought to be a lesson to +grave robbers!" "Muff Potter'll hang for this if they catch him!" This +was the drift of remark; and the minister said, "It was a judgment; His +hand is here." + +Now Tom shivered from head to heel; for his eye fell upon the stolid +face of Injun Joe. At this moment the crowd began to sway and struggle, +and voices shouted, "It's him! it's him! he's coming himself!" + +"Who? Who?" from twenty voices. + +"Muff Potter!" + +"Hallo, he's stopped!--Look out, he's turning! Don't let him get away!" + +People in the branches of the trees over Tom's head said he wasn't +trying to get away--he only looked doubtful and perplexed. + +"Infernal impudence!" said a bystander; "wanted to come and take a +quiet look at his work, I reckon--didn't expect any company." + +The crowd fell apart, now, and the Sheriff came through, +ostentatiously leading Potter by the arm. The poor fellow's face was +haggard, and his eyes showed the fear that was upon him. When he stood +before the murdered man, he shook as with a palsy, and he put his face +in his hands and burst into tears. + +"I didn't do it, friends," he sobbed; "'pon my word and honor I never +done it." + +"Who's accused you?" shouted a voice. + +This shot seemed to carry home. Potter lifted his face and looked +around him with a pathetic hopelessness in his eyes. He saw Injun Joe, +and exclaimed: + +"Oh, Injun Joe, you promised me you'd never--" + +"Is that your knife?" and it was thrust before him by the Sheriff. + +Potter would have fallen if they had not caught him and eased him to +the ground. Then he said: + +"Something told me 't if I didn't come back and get--" He shuddered; +then waved his nerveless hand with a vanquished gesture and said, "Tell +'em, Joe, tell 'em--it ain't any use any more." + +Then Huckleberry and Tom stood dumb and staring, and heard the +stony-hearted liar reel off his serene statement, they expecting every +moment that the clear sky would deliver God's lightnings upon his head, +and wondering to see how long the stroke was delayed. And when he had +finished and still stood alive and whole, their wavering impulse to +break their oath and save the poor betrayed prisoner's life faded and +vanished away, for plainly this miscreant had sold himself to Satan and +it would be fatal to meddle with the property of such a power as that. + +"Why didn't you leave? What did you want to come here for?" somebody +said. + +"I couldn't help it--I couldn't help it," Potter moaned. "I wanted to +run away, but I couldn't seem to come anywhere but here." And he fell +to sobbing again. + +Injun Joe repeated his statement, just as calmly, a few minutes +afterward on the inquest, under oath; and the boys, seeing that the +lightnings were still withheld, were confirmed in their belief that Joe +had sold himself to the devil. He was now become, to them, the most +balefully interesting object they had ever looked upon, and they could +not take their fascinated eyes from his face. + +They inwardly resolved to watch him nights, when opportunity should +offer, in the hope of getting a glimpse of his dread master. + +Injun Joe helped to raise the body of the murdered man and put it in a +wagon for removal; and it was whispered through the shuddering crowd +that the wound bled a little! The boys thought that this happy +circumstance would turn suspicion in the right direction; but they were +disappointed, for more than one villager remarked: + +"It was within three feet of Muff Potter when it done it." + +Tom's fearful secret and gnawing conscience disturbed his sleep for as +much as a week after this; and at breakfast one morning Sid said: + +"Tom, you pitch around and talk in your sleep so much that you keep me +awake half the time." + +Tom blanched and dropped his eyes. + +"It's a bad sign," said Aunt Polly, gravely. "What you got on your +mind, Tom?" + +"Nothing. Nothing 't I know of." But the boy's hand shook so that he +spilled his coffee. + +"And you do talk such stuff," Sid said. "Last night you said, 'It's +blood, it's blood, that's what it is!' You said that over and over. And +you said, 'Don't torment me so--I'll tell!' Tell WHAT? What is it +you'll tell?" + +Everything was swimming before Tom. There is no telling what might +have happened, now, but luckily the concern passed out of Aunt Polly's +face and she came to Tom's relief without knowing it. She said: + +"Sho! It's that dreadful murder. I dream about it most every night +myself. Sometimes I dream it's me that done it." + +Mary said she had been affected much the same way. Sid seemed +satisfied. Tom got out of the presence as quick as he plausibly could, +and after that he complained of toothache for a week, and tied up his +jaws every night. He never knew that Sid lay nightly watching, and +frequently slipped the bandage free and then leaned on his elbow +listening a good while at a time, and afterward slipped the bandage +back to its place again. Tom's distress of mind wore off gradually and +the toothache grew irksome and was discarded. If Sid really managed to +make anything out of Tom's disjointed mutterings, he kept it to himself. + +It seemed to Tom that his schoolmates never would get done holding +inquests on dead cats, and thus keeping his trouble present to his +mind. Sid noticed that Tom never was coroner at one of these inquiries, +though it had been his habit to take the lead in all new enterprises; +he noticed, too, that Tom never acted as a witness--and that was +strange; and Sid did not overlook the fact that Tom even showed a +marked aversion to these inquests, and always avoided them when he +could. Sid marvelled, but said nothing. However, even inquests went out +of vogue at last, and ceased to torture Tom's conscience. + +Every day or two, during this time of sorrow, Tom watched his +opportunity and went to the little grated jail-window and smuggled such +small comforts through to the "murderer" as he could get hold of. The +jail was a trifling little brick den that stood in a marsh at the edge +of the village, and no guards were afforded for it; indeed, it was +seldom occupied. These offerings greatly helped to ease Tom's +conscience. + +The villagers had a strong desire to tar-and-feather Injun Joe and +ride him on a rail, for body-snatching, but so formidable was his +character that nobody could be found who was willing to take the lead +in the matter, so it was dropped. He had been careful to begin both of +his inquest-statements with the fight, without confessing the +grave-robbery that preceded it; therefore it was deemed wisest not +to try the case in the courts at present. + + + +CHAPTER XII + +ONE of the reasons why Tom's mind had drifted away from its secret +troubles was, that it had found a new and weighty matter to interest +itself about. Becky Thatcher had stopped coming to school. Tom had +struggled with his pride a few days, and tried to "whistle her down the +wind," but failed. He began to find himself hanging around her father's +house, nights, and feeling very miserable. She was ill. What if she +should die! There was distraction in the thought. He no longer took an +interest in war, nor even in piracy. The charm of life was gone; there +was nothing but dreariness left. He put his hoop away, and his bat; +there was no joy in them any more. His aunt was concerned. She began to +try all manner of remedies on him. She was one of those people who are +infatuated with patent medicines and all new-fangled methods of +producing health or mending it. She was an inveterate experimenter in +these things. When something fresh in this line came out she was in a +fever, right away, to try it; not on herself, for she was never ailing, +but on anybody else that came handy. She was a subscriber for all the +"Health" periodicals and phrenological frauds; and the solemn ignorance +they were inflated with was breath to her nostrils. All the "rot" they +contained about ventilation, and how to go to bed, and how to get up, +and what to eat, and what to drink, and how much exercise to take, and +what frame of mind to keep one's self in, and what sort of clothing to +wear, was all gospel to her, and she never observed that her +health-journals of the current month customarily upset everything they +had recommended the month before. She was as simple-hearted and honest +as the day was long, and so she was an easy victim. She gathered +together her quack periodicals and her quack medicines, and thus armed +with death, went about on her pale horse, metaphorically speaking, with +"hell following after." But she never suspected that she was not an +angel of healing and the balm of Gilead in disguise, to the suffering +neighbors. + +The water treatment was new, now, and Tom's low condition was a +windfall to her. She had him out at daylight every morning, stood him +up in the woodshed and drowned him with a deluge of cold water; then +she scrubbed him down with a towel like a file, and so brought him to; +then she rolled him up in a wet sheet and put him away under blankets +till she sweated his soul clean and "the yellow stains of it came +through his pores"--as Tom said. + +Yet notwithstanding all this, the boy grew more and more melancholy +and pale and dejected. She added hot baths, sitz baths, shower baths, +and plunges. The boy remained as dismal as a hearse. She began to +assist the water with a slim oatmeal diet and blister-plasters. She +calculated his capacity as she would a jug's, and filled him up every +day with quack cure-alls. + +Tom had become indifferent to persecution by this time. This phase +filled the old lady's heart with consternation. This indifference must +be broken up at any cost. Now she heard of Pain-killer for the first +time. She ordered a lot at once. She tasted it and was filled with +gratitude. It was simply fire in a liquid form. She dropped the water +treatment and everything else, and pinned her faith to Pain-killer. She +gave Tom a teaspoonful and watched with the deepest anxiety for the +result. Her troubles were instantly at rest, her soul at peace again; +for the "indifference" was broken up. The boy could not have shown a +wilder, heartier interest, if she had built a fire under him. + +Tom felt that it was time to wake up; this sort of life might be +romantic enough, in his blighted condition, but it was getting to have +too little sentiment and too much distracting variety about it. So he +thought over various plans for relief, and finally hit pon that of +professing to be fond of Pain-killer. He asked for it so often that he +became a nuisance, and his aunt ended by telling him to help himself +and quit bothering her. If it had been Sid, she would have had no +misgivings to alloy her delight; but since it was Tom, she watched the +bottle clandestinely. She found that the medicine did really diminish, +but it did not occur to her that the boy was mending the health of a +crack in the sitting-room floor with it. + +One day Tom was in the act of dosing the crack when his aunt's yellow +cat came along, purring, eying the teaspoon avariciously, and begging +for a taste. Tom said: + +"Don't ask for it unless you want it, Peter." + +But Peter signified that he did want it. + +"You better make sure." + +Peter was sure. + +"Now you've asked for it, and I'll give it to you, because there ain't +anything mean about me; but if you find you don't like it, you mustn't +blame anybody but your own self." + +Peter was agreeable. So Tom pried his mouth open and poured down the +Pain-killer. Peter sprang a couple of yards in the air, and then +delivered a war-whoop and set off round and round the room, banging +against furniture, upsetting flower-pots, and making general havoc. +Next he rose on his hind feet and pranced around, in a frenzy of +enjoyment, with his head over his shoulder and his voice proclaiming +his unappeasable happiness. Then he went tearing around the house again +spreading chaos and destruction in his path. Aunt Polly entered in time +to see him throw a few double summersets, deliver a final mighty +hurrah, and sail through the open window, carrying the rest of the +flower-pots with him. The old lady stood petrified with astonishment, +peering over her glasses; Tom lay on the floor expiring with laughter. + +"Tom, what on earth ails that cat?" + +"I don't know, aunt," gasped the boy. + +"Why, I never see anything like it. What did make him act so?" + +"Deed I don't know, Aunt Polly; cats always act so when they're having +a good time." + +"They do, do they?" There was something in the tone that made Tom +apprehensive. + +"Yes'm. That is, I believe they do." + +"You DO?" + +"Yes'm." + +The old lady was bending down, Tom watching, with interest emphasized +by anxiety. Too late he divined her "drift." The handle of the telltale +teaspoon was visible under the bed-valance. Aunt Polly took it, held it +up. Tom winced, and dropped his eyes. Aunt Polly raised him by the +usual handle--his ear--and cracked his head soundly with her thimble. + +"Now, sir, what did you want to treat that poor dumb beast so, for?" + +"I done it out of pity for him--because he hadn't any aunt." + +"Hadn't any aunt!--you numskull. What has that got to do with it?" + +"Heaps. Because if he'd had one she'd a burnt him out herself! She'd a +roasted his bowels out of him 'thout any more feeling than if he was a +human!" + +Aunt Polly felt a sudden pang of remorse. This was putting the thing +in a new light; what was cruelty to a cat MIGHT be cruelty to a boy, +too. She began to soften; she felt sorry. Her eyes watered a little, +and she put her hand on Tom's head and said gently: + +"I was meaning for the best, Tom. And, Tom, it DID do you good." + +Tom looked up in her face with just a perceptible twinkle peeping +through his gravity. + +"I know you was meaning for the best, aunty, and so was I with Peter. +It done HIM good, too. I never see him get around so since--" + +"Oh, go 'long with you, Tom, before you aggravate me again. And you +try and see if you can't be a good boy, for once, and you needn't take +any more medicine." + +Tom reached school ahead of time. It was noticed that this strange +thing had been occurring every day latterly. And now, as usual of late, +he hung about the gate of the schoolyard instead of playing with his +comrades. He was sick, he said, and he looked it. He tried to seem to +be looking everywhere but whither he really was looking--down the road. +Presently Jeff Thatcher hove in sight, and Tom's face lighted; he gazed +a moment, and then turned sorrowfully away. When Jeff arrived, Tom +accosted him; and "led up" warily to opportunities for remark about +Becky, but the giddy lad never could see the bait. Tom watched and +watched, hoping whenever a frisking frock came in sight, and hating the +owner of it as soon as he saw she was not the right one. At last frocks +ceased to appear, and he dropped hopelessly into the dumps; he entered +the empty schoolhouse and sat down to suffer. Then one more frock +passed in at the gate, and Tom's heart gave a great bound. The next +instant he was out, and "going on" like an Indian; yelling, laughing, +chasing boys, jumping over the fence at risk of life and limb, throwing +handsprings, standing on his head--doing all the heroic things he could +conceive of, and keeping a furtive eye out, all the while, to see if +Becky Thatcher was noticing. But she seemed to be unconscious of it +all; she never looked. Could it be possible that she was not aware that +he was there? He carried his exploits to her immediate vicinity; came +war-whooping around, snatched a boy's cap, hurled it to the roof of the +schoolhouse, broke through a group of boys, tumbling them in every +direction, and fell sprawling, himself, under Becky's nose, almost +upsetting her--and she turned, with her nose in the air, and he heard +her say: "Mf! some people think they're mighty smart--always showing +off!" + +Tom's cheeks burned. He gathered himself up and sneaked off, crushed +and crestfallen. + + + +CHAPTER XIII + +TOM'S mind was made up now. He was gloomy and desperate. He was a +forsaken, friendless boy, he said; nobody loved him; when they found +out what they had driven him to, perhaps they would be sorry; he had +tried to do right and get along, but they would not let him; since +nothing would do them but to be rid of him, let it be so; and let them +blame HIM for the consequences--why shouldn't they? What right had the +friendless to complain? Yes, they had forced him to it at last: he +would lead a life of crime. There was no choice. + +By this time he was far down Meadow Lane, and the bell for school to +"take up" tinkled faintly upon his ear. He sobbed, now, to think he +should never, never hear that old familiar sound any more--it was very +hard, but it was forced on him; since he was driven out into the cold +world, he must submit--but he forgave them. Then the sobs came thick +and fast. + +Just at this point he met his soul's sworn comrade, Joe Harper +--hard-eyed, and with evidently a great and dismal purpose in his heart. +Plainly here were "two souls with but a single thought." Tom, wiping +his eyes with his sleeve, began to blubber out something about a +resolution to escape from hard usage and lack of sympathy at home by +roaming abroad into the great world never to return; and ended by +hoping that Joe would not forget him. + +But it transpired that this was a request which Joe had just been +going to make of Tom, and had come to hunt him up for that purpose. His +mother had whipped him for drinking some cream which he had never +tasted and knew nothing about; it was plain that she was tired of him +and wished him to go; if she felt that way, there was nothing for him +to do but succumb; he hoped she would be happy, and never regret having +driven her poor boy out into the unfeeling world to suffer and die. + +As the two boys walked sorrowing along, they made a new compact to +stand by each other and be brothers and never separate till death +relieved them of their troubles. Then they began to lay their plans. +Joe was for being a hermit, and living on crusts in a remote cave, and +dying, some time, of cold and want and grief; but after listening to +Tom, he conceded that there were some conspicuous advantages about a +life of crime, and so he consented to be a pirate. + +Three miles below St. Petersburg, at a point where the Mississippi +River was a trifle over a mile wide, there was a long, narrow, wooded +island, with a shallow bar at the head of it, and this offered well as +a rendezvous. It was not inhabited; it lay far over toward the further +shore, abreast a dense and almost wholly unpeopled forest. So Jackson's +Island was chosen. Who were to be the subjects of their piracies was a +matter that did not occur to them. Then they hunted up Huckleberry +Finn, and he joined them promptly, for all careers were one to him; he +was indifferent. They presently separated to meet at a lonely spot on +the river-bank two miles above the village at the favorite hour--which +was midnight. There was a small log raft there which they meant to +capture. Each would bring hooks and lines, and such provision as he +could steal in the most dark and mysterious way--as became outlaws. And +before the afternoon was done, they had all managed to enjoy the sweet +glory of spreading the fact that pretty soon the town would "hear +something." All who got this vague hint were cautioned to "be mum and +wait." + +About midnight Tom arrived with a boiled ham and a few trifles, +and stopped in a dense undergrowth on a small bluff overlooking the +meeting-place. It was starlight, and very still. The mighty river lay +like an ocean at rest. Tom listened a moment, but no sound disturbed the +quiet. Then he gave a low, distinct whistle. It was answered from under +the bluff. Tom whistled twice more; these signals were answered in the +same way. Then a guarded voice said: + +"Who goes there?" + +"Tom Sawyer, the Black Avenger of the Spanish Main. Name your names." + +"Huck Finn the Red-Handed, and Joe Harper the Terror of the Seas." Tom +had furnished these titles, from his favorite literature. + +"'Tis well. Give the countersign." + +Two hoarse whispers delivered the same awful word simultaneously to +the brooding night: + +"BLOOD!" + +Then Tom tumbled his ham over the bluff and let himself down after it, +tearing both skin and clothes to some extent in the effort. There was +an easy, comfortable path along the shore under the bluff, but it +lacked the advantages of difficulty and danger so valued by a pirate. + +The Terror of the Seas had brought a side of bacon, and had about worn +himself out with getting it there. Finn the Red-Handed had stolen a +skillet and a quantity of half-cured leaf tobacco, and had also brought +a few corn-cobs to make pipes with. But none of the pirates smoked or +"chewed" but himself. The Black Avenger of the Spanish Main said it +would never do to start without some fire. That was a wise thought; +matches were hardly known there in that day. They saw a fire +smouldering upon a great raft a hundred yards above, and they went +stealthily thither and helped themselves to a chunk. They made an +imposing adventure of it, saying, "Hist!" every now and then, and +suddenly halting with finger on lip; moving with hands on imaginary +dagger-hilts; and giving orders in dismal whispers that if "the foe" +stirred, to "let him have it to the hilt," because "dead men tell no +tales." They knew well enough that the raftsmen were all down at the +village laying in stores or having a spree, but still that was no +excuse for their conducting this thing in an unpiratical way. + +They shoved off, presently, Tom in command, Huck at the after oar and +Joe at the forward. Tom stood amidships, gloomy-browed, and with folded +arms, and gave his orders in a low, stern whisper: + +"Luff, and bring her to the wind!" + +"Aye-aye, sir!" + +"Steady, steady-y-y-y!" + +"Steady it is, sir!" + +"Let her go off a point!" + +"Point it is, sir!" + +As the boys steadily and monotonously drove the raft toward mid-stream +it was no doubt understood that these orders were given only for +"style," and were not intended to mean anything in particular. + +"What sail's she carrying?" + +"Courses, tops'ls, and flying-jib, sir." + +"Send the r'yals up! Lay out aloft, there, half a dozen of ye +--foretopmaststuns'l! Lively, now!" + +"Aye-aye, sir!" + +"Shake out that maintogalans'l! Sheets and braces! NOW my hearties!" + +"Aye-aye, sir!" + +"Hellum-a-lee--hard a port! Stand by to meet her when she comes! Port, +port! NOW, men! With a will! Stead-y-y-y!" + +"Steady it is, sir!" + +The raft drew beyond the middle of the river; the boys pointed her +head right, and then lay on their oars. The river was not high, so +there was not more than a two or three mile current. Hardly a word was +said during the next three-quarters of an hour. Now the raft was +passing before the distant town. Two or three glimmering lights showed +where it lay, peacefully sleeping, beyond the vague vast sweep of +star-gemmed water, unconscious of the tremendous event that was happening. +The Black Avenger stood still with folded arms, "looking his last" upon +the scene of his former joys and his later sufferings, and wishing +"she" could see him now, abroad on the wild sea, facing peril and death +with dauntless heart, going to his doom with a grim smile on his lips. +It was but a small strain on his imagination to remove Jackson's Island +beyond eyeshot of the village, and so he "looked his last" with a +broken and satisfied heart. The other pirates were looking their last, +too; and they all looked so long that they came near letting the +current drift them out of the range of the island. But they discovered +the danger in time, and made shift to avert it. About two o'clock in +the morning the raft grounded on the bar two hundred yards above the +head of the island, and they waded back and forth until they had landed +their freight. Part of the little raft's belongings consisted of an old +sail, and this they spread over a nook in the bushes for a tent to +shelter their provisions; but they themselves would sleep in the open +air in good weather, as became outlaws. + +They built a fire against the side of a great log twenty or thirty +steps within the sombre depths of the forest, and then cooked some +bacon in the frying-pan for supper, and used up half of the corn "pone" +stock they had brought. It seemed glorious sport to be feasting in that +wild, free way in the virgin forest of an unexplored and uninhabited +island, far from the haunts of men, and they said they never would +return to civilization. The climbing fire lit up their faces and threw +its ruddy glare upon the pillared tree-trunks of their forest temple, +and upon the varnished foliage and festooning vines. + +When the last crisp slice of bacon was gone, and the last allowance of +corn pone devoured, the boys stretched themselves out on the grass, +filled with contentment. They could have found a cooler place, but they +would not deny themselves such a romantic feature as the roasting +camp-fire. + +"AIN'T it gay?" said Joe. + +"It's NUTS!" said Tom. "What would the boys say if they could see us?" + +"Say? Well, they'd just die to be here--hey, Hucky!" + +"I reckon so," said Huckleberry; "anyways, I'm suited. I don't want +nothing better'n this. I don't ever get enough to eat, gen'ally--and +here they can't come and pick at a feller and bullyrag him so." + +"It's just the life for me," said Tom. "You don't have to get up, +mornings, and you don't have to go to school, and wash, and all that +blame foolishness. You see a pirate don't have to do ANYTHING, Joe, +when he's ashore, but a hermit HE has to be praying considerable, and +then he don't have any fun, anyway, all by himself that way." + +"Oh yes, that's so," said Joe, "but I hadn't thought much about it, +you know. I'd a good deal rather be a pirate, now that I've tried it." + +"You see," said Tom, "people don't go much on hermits, nowadays, like +they used to in old times, but a pirate's always respected. And a +hermit's got to sleep on the hardest place he can find, and put +sackcloth and ashes on his head, and stand out in the rain, and--" + +"What does he put sackcloth and ashes on his head for?" inquired Huck. + +"I dono. But they've GOT to do it. Hermits always do. You'd have to do +that if you was a hermit." + +"Dern'd if I would," said Huck. + +"Well, what would you do?" + +"I dono. But I wouldn't do that." + +"Why, Huck, you'd HAVE to. How'd you get around it?" + +"Why, I just wouldn't stand it. I'd run away." + +"Run away! Well, you WOULD be a nice old slouch of a hermit. You'd be +a disgrace." + +The Red-Handed made no response, being better employed. He had +finished gouging out a cob, and now he fitted a weed stem to it, loaded +it with tobacco, and was pressing a coal to the charge and blowing a +cloud of fragrant smoke--he was in the full bloom of luxurious +contentment. The other pirates envied him this majestic vice, and +secretly resolved to acquire it shortly. Presently Huck said: + +"What does pirates have to do?" + +Tom said: + +"Oh, they have just a bully time--take ships and burn them, and get +the money and bury it in awful places in their island where there's +ghosts and things to watch it, and kill everybody in the ships--make +'em walk a plank." + +"And they carry the women to the island," said Joe; "they don't kill +the women." + +"No," assented Tom, "they don't kill the women--they're too noble. And +the women's always beautiful, too. + +"And don't they wear the bulliest clothes! Oh no! All gold and silver +and di'monds," said Joe, with enthusiasm. + +"Who?" said Huck. + +"Why, the pirates." + +Huck scanned his own clothing forlornly. + +"I reckon I ain't dressed fitten for a pirate," said he, with a +regretful pathos in his voice; "but I ain't got none but these." + +But the other boys told him the fine clothes would come fast enough, +after they should have begun their adventures. They made him understand +that his poor rags would do to begin with, though it was customary for +wealthy pirates to start with a proper wardrobe. + +Gradually their talk died out and drowsiness began to steal upon the +eyelids of the little waifs. The pipe dropped from the fingers of the +Red-Handed, and he slept the sleep of the conscience-free and the +weary. The Terror of the Seas and the Black Avenger of the Spanish Main +had more difficulty in getting to sleep. They said their prayers +inwardly, and lying down, since there was nobody there with authority +to make them kneel and recite aloud; in truth, they had a mind not to +say them at all, but they were afraid to proceed to such lengths as +that, lest they might call down a sudden and special thunderbolt from +heaven. Then at once they reached and hovered upon the imminent verge +of sleep--but an intruder came, now, that would not "down." It was +conscience. They began to feel a vague fear that they had been doing +wrong to run away; and next they thought of the stolen meat, and then +the real torture came. They tried to argue it away by reminding +conscience that they had purloined sweetmeats and apples scores of +times; but conscience was not to be appeased by such thin +plausibilities; it seemed to them, in the end, that there was no +getting around the stubborn fact that taking sweetmeats was only +"hooking," while taking bacon and hams and such valuables was plain +simple stealing--and there was a command against that in the Bible. So +they inwardly resolved that so long as they remained in the business, +their piracies should not again be sullied with the crime of stealing. +Then conscience granted a truce, and these curiously inconsistent +pirates fell peacefully to sleep. + + + +CHAPTER XIV + +WHEN Tom awoke in the morning, he wondered where he was. He sat up and +rubbed his eyes and looked around. Then he comprehended. It was the +cool gray dawn, and there was a delicious sense of repose and peace in +the deep pervading calm and silence of the woods. Not a leaf stirred; +not a sound obtruded upon great Nature's meditation. Beaded dewdrops +stood upon the leaves and grasses. A white layer of ashes covered the +fire, and a thin blue breath of smoke rose straight into the air. Joe +and Huck still slept. + +Now, far away in the woods a bird called; another answered; presently +the hammering of a woodpecker was heard. Gradually the cool dim gray of +the morning whitened, and as gradually sounds multiplied and life +manifested itself. The marvel of Nature shaking off sleep and going to +work unfolded itself to the musing boy. A little green worm came +crawling over a dewy leaf, lifting two-thirds of his body into the air +from time to time and "sniffing around," then proceeding again--for he +was measuring, Tom said; and when the worm approached him, of its own +accord, he sat as still as a stone, with his hopes rising and falling, +by turns, as the creature still came toward him or seemed inclined to +go elsewhere; and when at last it considered a painful moment with its +curved body in the air and then came decisively down upon Tom's leg and +began a journey over him, his whole heart was glad--for that meant that +he was going to have a new suit of clothes--without the shadow of a +doubt a gaudy piratical uniform. Now a procession of ants appeared, +from nowhere in particular, and went about their labors; one struggled +manfully by with a dead spider five times as big as itself in its arms, +and lugged it straight up a tree-trunk. A brown spotted lady-bug +climbed the dizzy height of a grass blade, and Tom bent down close to +it and said, "Lady-bug, lady-bug, fly away home, your house is on fire, +your children's alone," and she took wing and went off to see about it +--which did not surprise the boy, for he knew of old that this insect was +credulous about conflagrations, and he had practised upon its +simplicity more than once. A tumblebug came next, heaving sturdily at +its ball, and Tom touched the creature, to see it shut its legs against +its body and pretend to be dead. The birds were fairly rioting by this +time. A catbird, the Northern mocker, lit in a tree over Tom's head, +and trilled out her imitations of her neighbors in a rapture of +enjoyment; then a shrill jay swept down, a flash of blue flame, and +stopped on a twig almost within the boy's reach, cocked his head to one +side and eyed the strangers with a consuming curiosity; a gray squirrel +and a big fellow of the "fox" kind came skurrying along, sitting up at +intervals to inspect and chatter at the boys, for the wild things had +probably never seen a human being before and scarcely knew whether to +be afraid or not. All Nature was wide awake and stirring, now; long +lances of sunlight pierced down through the dense foliage far and near, +and a few butterflies came fluttering upon the scene. + +Tom stirred up the other pirates and they all clattered away with a +shout, and in a minute or two were stripped and chasing after and +tumbling over each other in the shallow limpid water of the white +sandbar. They felt no longing for the little village sleeping in the +distance beyond the majestic waste of water. A vagrant current or a +slight rise in the river had carried off their raft, but this only +gratified them, since its going was something like burning the bridge +between them and civilization. + +They came back to camp wonderfully refreshed, glad-hearted, and +ravenous; and they soon had the camp-fire blazing up again. Huck found +a spring of clear cold water close by, and the boys made cups of broad +oak or hickory leaves, and felt that water, sweetened with such a +wildwood charm as that, would be a good enough substitute for coffee. +While Joe was slicing bacon for breakfast, Tom and Huck asked him to +hold on a minute; they stepped to a promising nook in the river-bank +and threw in their lines; almost immediately they had reward. Joe had +not had time to get impatient before they were back again with some +handsome bass, a couple of sun-perch and a small catfish--provisions +enough for quite a family. They fried the fish with the bacon, and were +astonished; for no fish had ever seemed so delicious before. They did +not know that the quicker a fresh-water fish is on the fire after he is +caught the better he is; and they reflected little upon what a sauce +open-air sleeping, open-air exercise, bathing, and a large ingredient +of hunger make, too. + +They lay around in the shade, after breakfast, while Huck had a smoke, +and then went off through the woods on an exploring expedition. They +tramped gayly along, over decaying logs, through tangled underbrush, +among solemn monarchs of the forest, hung from their crowns to the +ground with a drooping regalia of grape-vines. Now and then they came +upon snug nooks carpeted with grass and jeweled with flowers. + +They found plenty of things to be delighted with, but nothing to be +astonished at. They discovered that the island was about three miles +long and a quarter of a mile wide, and that the shore it lay closest to +was only separated from it by a narrow channel hardly two hundred yards +wide. They took a swim about every hour, so it was close upon the +middle of the afternoon when they got back to camp. They were too +hungry to stop to fish, but they fared sumptuously upon cold ham, and +then threw themselves down in the shade to talk. But the talk soon +began to drag, and then died. The stillness, the solemnity that brooded +in the woods, and the sense of loneliness, began to tell upon the +spirits of the boys. They fell to thinking. A sort of undefined longing +crept upon them. This took dim shape, presently--it was budding +homesickness. Even Finn the Red-Handed was dreaming of his doorsteps +and empty hogsheads. But they were all ashamed of their weakness, and +none was brave enough to speak his thought. + +For some time, now, the boys had been dully conscious of a peculiar +sound in the distance, just as one sometimes is of the ticking of a +clock which he takes no distinct note of. But now this mysterious sound +became more pronounced, and forced a recognition. The boys started, +glanced at each other, and then each assumed a listening attitude. +There was a long silence, profound and unbroken; then a deep, sullen +boom came floating down out of the distance. + +"What is it!" exclaimed Joe, under his breath. + +"I wonder," said Tom in a whisper. + +"'Tain't thunder," said Huckleberry, in an awed tone, "becuz thunder--" + +"Hark!" said Tom. "Listen--don't talk." + +They waited a time that seemed an age, and then the same muffled boom +troubled the solemn hush. + +"Let's go and see." + +They sprang to their feet and hurried to the shore toward the town. +They parted the bushes on the bank and peered out over the water. The +little steam ferryboat was about a mile below the village, drifting +with the current. Her broad deck seemed crowded with people. There were +a great many skiffs rowing about or floating with the stream in the +neighborhood of the ferryboat, but the boys could not determine what +the men in them were doing. Presently a great jet of white smoke burst +from the ferryboat's side, and as it expanded and rose in a lazy cloud, +that same dull throb of sound was borne to the listeners again. + +"I know now!" exclaimed Tom; "somebody's drownded!" + +"That's it!" said Huck; "they done that last summer, when Bill Turner +got drownded; they shoot a cannon over the water, and that makes him +come up to the top. Yes, and they take loaves of bread and put +quicksilver in 'em and set 'em afloat, and wherever there's anybody +that's drownded, they'll float right there and stop." + +"Yes, I've heard about that," said Joe. "I wonder what makes the bread +do that." + +"Oh, it ain't the bread, so much," said Tom; "I reckon it's mostly +what they SAY over it before they start it out." + +"But they don't say anything over it," said Huck. "I've seen 'em and +they don't." + +"Well, that's funny," said Tom. "But maybe they say it to themselves. +Of COURSE they do. Anybody might know that." + +The other boys agreed that there was reason in what Tom said, because +an ignorant lump of bread, uninstructed by an incantation, could not be +expected to act very intelligently when set upon an errand of such +gravity. + +"By jings, I wish I was over there, now," said Joe. + +"I do too" said Huck "I'd give heaps to know who it is." + +The boys still listened and watched. Presently a revealing thought +flashed through Tom's mind, and he exclaimed: + +"Boys, I know who's drownded--it's us!" + +They felt like heroes in an instant. Here was a gorgeous triumph; they +were missed; they were mourned; hearts were breaking on their account; +tears were being shed; accusing memories of unkindness to these poor +lost lads were rising up, and unavailing regrets and remorse were being +indulged; and best of all, the departed were the talk of the whole +town, and the envy of all the boys, as far as this dazzling notoriety +was concerned. This was fine. It was worth while to be a pirate, after +all. + +As twilight drew on, the ferryboat went back to her accustomed +business and the skiffs disappeared. The pirates returned to camp. They +were jubilant with vanity over their new grandeur and the illustrious +trouble they were making. They caught fish, cooked supper and ate it, +and then fell to guessing at what the village was thinking and saying +about them; and the pictures they drew of the public distress on their +account were gratifying to look upon--from their point of view. But +when the shadows of night closed them in, they gradually ceased to +talk, and sat gazing into the fire, with their minds evidently +wandering elsewhere. The excitement was gone, now, and Tom and Joe +could not keep back thoughts of certain persons at home who were not +enjoying this fine frolic as much as they were. Misgivings came; they +grew troubled and unhappy; a sigh or two escaped, unawares. By and by +Joe timidly ventured upon a roundabout "feeler" as to how the others +might look upon a return to civilization--not right now, but-- + +Tom withered him with derision! Huck, being uncommitted as yet, joined +in with Tom, and the waverer quickly "explained," and was glad to get +out of the scrape with as little taint of chicken-hearted homesickness +clinging to his garments as he could. Mutiny was effectually laid to +rest for the moment. + +As the night deepened, Huck began to nod, and presently to snore. Joe +followed next. Tom lay upon his elbow motionless, for some time, +watching the two intently. At last he got up cautiously, on his knees, +and went searching among the grass and the flickering reflections flung +by the camp-fire. He picked up and inspected several large +semi-cylinders of the thin white bark of a sycamore, and finally chose +two which seemed to suit him. Then he knelt by the fire and painfully +wrote something upon each of these with his "red keel"; one he rolled up +and put in his jacket pocket, and the other he put in Joe's hat and +removed it to a little distance from the owner. And he also put into the +hat certain schoolboy treasures of almost inestimable value--among them +a lump of chalk, an India-rubber ball, three fishhooks, and one of that +kind of marbles known as a "sure 'nough crystal." Then he tiptoed his +way cautiously among the trees till he felt that he was out of hearing, +and straightway broke into a keen run in the direction of the sandbar. + + + +CHAPTER XV + +A FEW minutes later Tom was in the shoal water of the bar, wading +toward the Illinois shore. Before the depth reached his middle he was +half-way over; the current would permit no more wading, now, so he +struck out confidently to swim the remaining hundred yards. He swam +quartering upstream, but still was swept downward rather faster than he +had expected. However, he reached the shore finally, and drifted along +till he found a low place and drew himself out. He put his hand on his +jacket pocket, found his piece of bark safe, and then struck through +the woods, following the shore, with streaming garments. Shortly before +ten o'clock he came out into an open place opposite the village, and +saw the ferryboat lying in the shadow of the trees and the high bank. +Everything was quiet under the blinking stars. He crept down the bank, +watching with all his eyes, slipped into the water, swam three or four +strokes and climbed into the skiff that did "yawl" duty at the boat's +stern. He laid himself down under the thwarts and waited, panting. + +Presently the cracked bell tapped and a voice gave the order to "cast +off." A minute or two later the skiff's head was standing high up, +against the boat's swell, and the voyage was begun. Tom felt happy in +his success, for he knew it was the boat's last trip for the night. At +the end of a long twelve or fifteen minutes the wheels stopped, and Tom +slipped overboard and swam ashore in the dusk, landing fifty yards +downstream, out of danger of possible stragglers. + +He flew along unfrequented alleys, and shortly found himself at his +aunt's back fence. He climbed over, approached the "ell," and looked in +at the sitting-room window, for a light was burning there. There sat +Aunt Polly, Sid, Mary, and Joe Harper's mother, grouped together, +talking. They were by the bed, and the bed was between them and the +door. Tom went to the door and began to softly lift the latch; then he +pressed gently and the door yielded a crack; he continued pushing +cautiously, and quaking every time it creaked, till he judged he might +squeeze through on his knees; so he put his head through and began, +warily. + +"What makes the candle blow so?" said Aunt Polly. Tom hurried up. +"Why, that door's open, I believe. Why, of course it is. No end of +strange things now. Go 'long and shut it, Sid." + +Tom disappeared under the bed just in time. He lay and "breathed" +himself for a time, and then crept to where he could almost touch his +aunt's foot. + +"But as I was saying," said Aunt Polly, "he warn't BAD, so to say +--only mischEEvous. Only just giddy, and harum-scarum, you know. He +warn't any more responsible than a colt. HE never meant any harm, and +he was the best-hearted boy that ever was"--and she began to cry. + +"It was just so with my Joe--always full of his devilment, and up to +every kind of mischief, but he was just as unselfish and kind as he +could be--and laws bless me, to think I went and whipped him for taking +that cream, never once recollecting that I throwed it out myself +because it was sour, and I never to see him again in this world, never, +never, never, poor abused boy!" And Mrs. Harper sobbed as if her heart +would break. + +"I hope Tom's better off where he is," said Sid, "but if he'd been +better in some ways--" + +"SID!" Tom felt the glare of the old lady's eye, though he could not +see it. "Not a word against my Tom, now that he's gone! God'll take +care of HIM--never you trouble YOURself, sir! Oh, Mrs. Harper, I don't +know how to give him up! I don't know how to give him up! He was such a +comfort to me, although he tormented my old heart out of me, 'most." + +"The Lord giveth and the Lord hath taken away--Blessed be the name of +the Lord! But it's so hard--Oh, it's so hard! Only last Saturday my +Joe busted a firecracker right under my nose and I knocked him +sprawling. Little did I know then, how soon--Oh, if it was to do over +again I'd hug him and bless him for it." + +"Yes, yes, yes, I know just how you feel, Mrs. Harper, I know just +exactly how you feel. No longer ago than yesterday noon, my Tom took +and filled the cat full of Pain-killer, and I did think the cretur +would tear the house down. And God forgive me, I cracked Tom's head +with my thimble, poor boy, poor dead boy. But he's out of all his +troubles now. And the last words I ever heard him say was to reproach--" + +But this memory was too much for the old lady, and she broke entirely +down. Tom was snuffling, now, himself--and more in pity of himself than +anybody else. He could hear Mary crying, and putting in a kindly word +for him from time to time. He began to have a nobler opinion of himself +than ever before. Still, he was sufficiently touched by his aunt's +grief to long to rush out from under the bed and overwhelm her with +joy--and the theatrical gorgeousness of the thing appealed strongly to +his nature, too, but he resisted and lay still. + +He went on listening, and gathered by odds and ends that it was +conjectured at first that the boys had got drowned while taking a swim; +then the small raft had been missed; next, certain boys said the +missing lads had promised that the village should "hear something" +soon; the wise-heads had "put this and that together" and decided that +the lads had gone off on that raft and would turn up at the next town +below, presently; but toward noon the raft had been found, lodged +against the Missouri shore some five or six miles below the village +--and then hope perished; they must be drowned, else hunger would have +driven them home by nightfall if not sooner. It was believed that the +search for the bodies had been a fruitless effort merely because the +drowning must have occurred in mid-channel, since the boys, being good +swimmers, would otherwise have escaped to shore. This was Wednesday +night. If the bodies continued missing until Sunday, all hope would be +given over, and the funerals would be preached on that morning. Tom +shuddered. + +Mrs. Harper gave a sobbing good-night and turned to go. Then with a +mutual impulse the two bereaved women flung themselves into each +other's arms and had a good, consoling cry, and then parted. Aunt Polly +was tender far beyond her wont, in her good-night to Sid and Mary. Sid +snuffled a bit and Mary went off crying with all her heart. + +Aunt Polly knelt down and prayed for Tom so touchingly, so +appealingly, and with such measureless love in her words and her old +trembling voice, that he was weltering in tears again, long before she +was through. + +He had to keep still long after she went to bed, for she kept making +broken-hearted ejaculations from time to time, tossing unrestfully, and +turning over. But at last she was still, only moaning a little in her +sleep. Now the boy stole out, rose gradually by the bedside, shaded the +candle-light with his hand, and stood regarding her. His heart was full +of pity for her. He took out his sycamore scroll and placed it by the +candle. But something occurred to him, and he lingered considering. His +face lighted with a happy solution of his thought; he put the bark +hastily in his pocket. Then he bent over and kissed the faded lips, and +straightway made his stealthy exit, latching the door behind him. + +He threaded his way back to the ferry landing, found nobody at large +there, and walked boldly on board the boat, for he knew she was +tenantless except that there was a watchman, who always turned in and +slept like a graven image. He untied the skiff at the stern, slipped +into it, and was soon rowing cautiously upstream. When he had pulled a +mile above the village, he started quartering across and bent himself +stoutly to his work. He hit the landing on the other side neatly, for +this was a familiar bit of work to him. He was moved to capture the +skiff, arguing that it might be considered a ship and therefore +legitimate prey for a pirate, but he knew a thorough search would be +made for it and that might end in revelations. So he stepped ashore and +entered the woods. + +He sat down and took a long rest, torturing himself meanwhile to keep +awake, and then started warily down the home-stretch. The night was far +spent. It was broad daylight before he found himself fairly abreast the +island bar. He rested again until the sun was well up and gilding the +great river with its splendor, and then he plunged into the stream. A +little later he paused, dripping, upon the threshold of the camp, and +heard Joe say: + +"No, Tom's true-blue, Huck, and he'll come back. He won't desert. He +knows that would be a disgrace to a pirate, and Tom's too proud for +that sort of thing. He's up to something or other. Now I wonder what?" + +"Well, the things is ours, anyway, ain't they?" + +"Pretty near, but not yet, Huck. The writing says they are if he ain't +back here to breakfast." + +"Which he is!" exclaimed Tom, with fine dramatic effect, stepping +grandly into camp. + +A sumptuous breakfast of bacon and fish was shortly provided, and as +the boys set to work upon it, Tom recounted (and adorned) his +adventures. They were a vain and boastful company of heroes when the +tale was done. Then Tom hid himself away in a shady nook to sleep till +noon, and the other pirates got ready to fish and explore. + + + +CHAPTER XVI + +AFTER dinner all the gang turned out to hunt for turtle eggs on the +bar. They went about poking sticks into the sand, and when they found a +soft place they went down on their knees and dug with their hands. +Sometimes they would take fifty or sixty eggs out of one hole. They +were perfectly round white things a trifle smaller than an English +walnut. They had a famous fried-egg feast that night, and another on +Friday morning. + +After breakfast they went whooping and prancing out on the bar, and +chased each other round and round, shedding clothes as they went, until +they were naked, and then continued the frolic far away up the shoal +water of the bar, against the stiff current, which latter tripped their +legs from under them from time to time and greatly increased the fun. +And now and then they stooped in a group and splashed water in each +other's faces with their palms, gradually approaching each other, with +averted faces to avoid the strangling sprays, and finally gripping and +struggling till the best man ducked his neighbor, and then they all +went under in a tangle of white legs and arms and came up blowing, +sputtering, laughing, and gasping for breath at one and the same time. + +When they were well exhausted, they would run out and sprawl on the +dry, hot sand, and lie there and cover themselves up with it, and by +and by break for the water again and go through the original +performance once more. Finally it occurred to them that their naked +skin represented flesh-colored "tights" very fairly; so they drew a +ring in the sand and had a circus--with three clowns in it, for none +would yield this proudest post to his neighbor. + +Next they got their marbles and played "knucks" and "ring-taw" and +"keeps" till that amusement grew stale. Then Joe and Huck had another +swim, but Tom would not venture, because he found that in kicking off +his trousers he had kicked his string of rattlesnake rattles off his +ankle, and he wondered how he had escaped cramp so long without the +protection of this mysterious charm. He did not venture again until he +had found it, and by that time the other boys were tired and ready to +rest. They gradually wandered apart, dropped into the "dumps," and fell +to gazing longingly across the wide river to where the village lay +drowsing in the sun. Tom found himself writing "BECKY" in the sand with +his big toe; he scratched it out, and was angry with himself for his +weakness. But he wrote it again, nevertheless; he could not help it. He +erased it once more and then took himself out of temptation by driving +the other boys together and joining them. + +But Joe's spirits had gone down almost beyond resurrection. He was so +homesick that he could hardly endure the misery of it. The tears lay +very near the surface. Huck was melancholy, too. Tom was downhearted, +but tried hard not to show it. He had a secret which he was not ready +to tell, yet, but if this mutinous depression was not broken up soon, +he would have to bring it out. He said, with a great show of +cheerfulness: + +"I bet there's been pirates on this island before, boys. We'll explore +it again. They've hid treasures here somewhere. How'd you feel to light +on a rotten chest full of gold and silver--hey?" + +But it roused only faint enthusiasm, which faded out, with no reply. +Tom tried one or two other seductions; but they failed, too. It was +discouraging work. Joe sat poking up the sand with a stick and looking +very gloomy. Finally he said: + +"Oh, boys, let's give it up. I want to go home. It's so lonesome." + +"Oh no, Joe, you'll feel better by and by," said Tom. "Just think of +the fishing that's here." + +"I don't care for fishing. I want to go home." + +"But, Joe, there ain't such another swimming-place anywhere." + +"Swimming's no good. I don't seem to care for it, somehow, when there +ain't anybody to say I sha'n't go in. I mean to go home." + +"Oh, shucks! Baby! You want to see your mother, I reckon." + +"Yes, I DO want to see my mother--and you would, too, if you had one. +I ain't any more baby than you are." And Joe snuffled a little. + +"Well, we'll let the cry-baby go home to his mother, won't we, Huck? +Poor thing--does it want to see its mother? And so it shall. You like +it here, don't you, Huck? We'll stay, won't we?" + +Huck said, "Y-e-s"--without any heart in it. + +"I'll never speak to you again as long as I live," said Joe, rising. +"There now!" And he moved moodily away and began to dress himself. + +"Who cares!" said Tom. "Nobody wants you to. Go 'long home and get +laughed at. Oh, you're a nice pirate. Huck and me ain't cry-babies. +We'll stay, won't we, Huck? Let him go if he wants to. I reckon we can +get along without him, per'aps." + +But Tom was uneasy, nevertheless, and was alarmed to see Joe go +sullenly on with his dressing. And then it was discomforting to see +Huck eying Joe's preparations so wistfully, and keeping up such an +ominous silence. Presently, without a parting word, Joe began to wade +off toward the Illinois shore. Tom's heart began to sink. He glanced at +Huck. Huck could not bear the look, and dropped his eyes. Then he said: + +"I want to go, too, Tom. It was getting so lonesome anyway, and now +it'll be worse. Let's us go, too, Tom." + +"I won't! You can all go, if you want to. I mean to stay." + +"Tom, I better go." + +"Well, go 'long--who's hendering you." + +Huck began to pick up his scattered clothes. He said: + +"Tom, I wisht you'd come, too. Now you think it over. We'll wait for +you when we get to shore." + +"Well, you'll wait a blame long time, that's all." + +Huck started sorrowfully away, and Tom stood looking after him, with a +strong desire tugging at his heart to yield his pride and go along too. +He hoped the boys would stop, but they still waded slowly on. It +suddenly dawned on Tom that it was become very lonely and still. He +made one final struggle with his pride, and then darted after his +comrades, yelling: + +"Wait! Wait! I want to tell you something!" + +They presently stopped and turned around. When he got to where they +were, he began unfolding his secret, and they listened moodily till at +last they saw the "point" he was driving at, and then they set up a +war-whoop of applause and said it was "splendid!" and said if he had +told them at first, they wouldn't have started away. He made a plausible +excuse; but his real reason had been the fear that not even the secret +would keep them with him any very great length of time, and so he had +meant to hold it in reserve as a last seduction. + +The lads came gayly back and went at their sports again with a will, +chattering all the time about Tom's stupendous plan and admiring the +genius of it. After a dainty egg and fish dinner, Tom said he wanted to +learn to smoke, now. Joe caught at the idea and said he would like to +try, too. So Huck made pipes and filled them. These novices had never +smoked anything before but cigars made of grape-vine, and they "bit" +the tongue, and were not considered manly anyway. + +Now they stretched themselves out on their elbows and began to puff, +charily, and with slender confidence. The smoke had an unpleasant +taste, and they gagged a little, but Tom said: + +"Why, it's just as easy! If I'd a knowed this was all, I'd a learnt +long ago." + +"So would I," said Joe. "It's just nothing." + +"Why, many a time I've looked at people smoking, and thought well I +wish I could do that; but I never thought I could," said Tom. + +"That's just the way with me, hain't it, Huck? You've heard me talk +just that way--haven't you, Huck? I'll leave it to Huck if I haven't." + +"Yes--heaps of times," said Huck. + +"Well, I have too," said Tom; "oh, hundreds of times. Once down by the +slaughter-house. Don't you remember, Huck? Bob Tanner was there, and +Johnny Miller, and Jeff Thatcher, when I said it. Don't you remember, +Huck, 'bout me saying that?" + +"Yes, that's so," said Huck. "That was the day after I lost a white +alley. No, 'twas the day before." + +"There--I told you so," said Tom. "Huck recollects it." + +"I bleeve I could smoke this pipe all day," said Joe. "I don't feel +sick." + +"Neither do I," said Tom. "I could smoke it all day. But I bet you +Jeff Thatcher couldn't." + +"Jeff Thatcher! Why, he'd keel over just with two draws. Just let him +try it once. HE'D see!" + +"I bet he would. And Johnny Miller--I wish could see Johnny Miller +tackle it once." + +"Oh, don't I!" said Joe. "Why, I bet you Johnny Miller couldn't any +more do this than nothing. Just one little snifter would fetch HIM." + +"'Deed it would, Joe. Say--I wish the boys could see us now." + +"So do I." + +"Say--boys, don't say anything about it, and some time when they're +around, I'll come up to you and say, 'Joe, got a pipe? I want a smoke.' +And you'll say, kind of careless like, as if it warn't anything, you'll +say, 'Yes, I got my OLD pipe, and another one, but my tobacker ain't +very good.' And I'll say, 'Oh, that's all right, if it's STRONG +enough.' And then you'll out with the pipes, and we'll light up just as +ca'm, and then just see 'em look!" + +"By jings, that'll be gay, Tom! I wish it was NOW!" + +"So do I! And when we tell 'em we learned when we was off pirating, +won't they wish they'd been along?" + +"Oh, I reckon not! I'll just BET they will!" + +So the talk ran on. But presently it began to flag a trifle, and grow +disjointed. The silences widened; the expectoration marvellously +increased. Every pore inside the boys' cheeks became a spouting +fountain; they could scarcely bail out the cellars under their tongues +fast enough to prevent an inundation; little overflowings down their +throats occurred in spite of all they could do, and sudden retchings +followed every time. Both boys were looking very pale and miserable, +now. Joe's pipe dropped from his nerveless fingers. Tom's followed. +Both fountains were going furiously and both pumps bailing with might +and main. Joe said feebly: + +"I've lost my knife. I reckon I better go and find it." + +Tom said, with quivering lips and halting utterance: + +"I'll help you. You go over that way and I'll hunt around by the +spring. No, you needn't come, Huck--we can find it." + +So Huck sat down again, and waited an hour. Then he found it lonesome, +and went to find his comrades. They were wide apart in the woods, both +very pale, both fast asleep. But something informed him that if they +had had any trouble they had got rid of it. + +They were not talkative at supper that night. They had a humble look, +and when Huck prepared his pipe after the meal and was going to prepare +theirs, they said no, they were not feeling very well--something they +ate at dinner had disagreed with them. + +About midnight Joe awoke, and called the boys. There was a brooding +oppressiveness in the air that seemed to bode something. The boys +huddled themselves together and sought the friendly companionship of +the fire, though the dull dead heat of the breathless atmosphere was +stifling. They sat still, intent and waiting. The solemn hush +continued. Beyond the light of the fire everything was swallowed up in +the blackness of darkness. Presently there came a quivering glow that +vaguely revealed the foliage for a moment and then vanished. By and by +another came, a little stronger. Then another. Then a faint moan came +sighing through the branches of the forest and the boys felt a fleeting +breath upon their cheeks, and shuddered with the fancy that the Spirit +of the Night had gone by. There was a pause. Now a weird flash turned +night into day and showed every little grass-blade, separate and +distinct, that grew about their feet. And it showed three white, +startled faces, too. A deep peal of thunder went rolling and tumbling +down the heavens and lost itself in sullen rumblings in the distance. A +sweep of chilly air passed by, rustling all the leaves and snowing the +flaky ashes broadcast about the fire. Another fierce glare lit up the +forest and an instant crash followed that seemed to rend the tree-tops +right over the boys' heads. They clung together in terror, in the thick +gloom that followed. A few big rain-drops fell pattering upon the +leaves. + +"Quick! boys, go for the tent!" exclaimed Tom. + +They sprang away, stumbling over roots and among vines in the dark, no +two plunging in the same direction. A furious blast roared through the +trees, making everything sing as it went. One blinding flash after +another came, and peal on peal of deafening thunder. And now a +drenching rain poured down and the rising hurricane drove it in sheets +along the ground. The boys cried out to each other, but the roaring +wind and the booming thunder-blasts drowned their voices utterly. +However, one by one they straggled in at last and took shelter under +the tent, cold, scared, and streaming with water; but to have company +in misery seemed something to be grateful for. They could not talk, the +old sail flapped so furiously, even if the other noises would have +allowed them. The tempest rose higher and higher, and presently the +sail tore loose from its fastenings and went winging away on the blast. +The boys seized each others' hands and fled, with many tumblings and +bruises, to the shelter of a great oak that stood upon the river-bank. +Now the battle was at its highest. Under the ceaseless conflagration of +lightning that flamed in the skies, everything below stood out in +clean-cut and shadowless distinctness: the bending trees, the billowy +river, white with foam, the driving spray of spume-flakes, the dim +outlines of the high bluffs on the other side, glimpsed through the +drifting cloud-rack and the slanting veil of rain. Every little while +some giant tree yielded the fight and fell crashing through the younger +growth; and the unflagging thunder-peals came now in ear-splitting +explosive bursts, keen and sharp, and unspeakably appalling. The storm +culminated in one matchless effort that seemed likely to tear the island +to pieces, burn it up, drown it to the tree-tops, blow it away, and +deafen every creature in it, all at one and the same moment. It was a +wild night for homeless young heads to be out in. + +But at last the battle was done, and the forces retired with weaker +and weaker threatenings and grumblings, and peace resumed her sway. The +boys went back to camp, a good deal awed; but they found there was +still something to be thankful for, because the great sycamore, the +shelter of their beds, was a ruin, now, blasted by the lightnings, and +they were not under it when the catastrophe happened. + +Everything in camp was drenched, the camp-fire as well; for they were +but heedless lads, like their generation, and had made no provision +against rain. Here was matter for dismay, for they were soaked through +and chilled. They were eloquent in their distress; but they presently +discovered that the fire had eaten so far up under the great log it had +been built against (where it curved upward and separated itself from +the ground), that a handbreadth or so of it had escaped wetting; so +they patiently wrought until, with shreds and bark gathered from the +under sides of sheltered logs, they coaxed the fire to burn again. Then +they piled on great dead boughs till they had a roaring furnace, and +were glad-hearted once more. They dried their boiled ham and had a +feast, and after that they sat by the fire and expanded and glorified +their midnight adventure until morning, for there was not a dry spot to +sleep on, anywhere around. + +As the sun began to steal in upon the boys, drowsiness came over them, +and they went out on the sandbar and lay down to sleep. They got +scorched out by and by, and drearily set about getting breakfast. After +the meal they felt rusty, and stiff-jointed, and a little homesick once +more. Tom saw the signs, and fell to cheering up the pirates as well as +he could. But they cared nothing for marbles, or circus, or swimming, +or anything. He reminded them of the imposing secret, and raised a ray +of cheer. While it lasted, he got them interested in a new device. This +was to knock off being pirates, for a while, and be Indians for a +change. They were attracted by this idea; so it was not long before +they were stripped, and striped from head to heel with black mud, like +so many zebras--all of them chiefs, of course--and then they went +tearing through the woods to attack an English settlement. + +By and by they separated into three hostile tribes, and darted upon +each other from ambush with dreadful war-whoops, and killed and scalped +each other by thousands. It was a gory day. Consequently it was an +extremely satisfactory one. + +They assembled in camp toward supper-time, hungry and happy; but now a +difficulty arose--hostile Indians could not break the bread of +hospitality together without first making peace, and this was a simple +impossibility without smoking a pipe of peace. There was no other +process that ever they had heard of. Two of the savages almost wished +they had remained pirates. However, there was no other way; so with +such show of cheerfulness as they could muster they called for the pipe +and took their whiff as it passed, in due form. + +And behold, they were glad they had gone into savagery, for they had +gained something; they found that they could now smoke a little without +having to go and hunt for a lost knife; they did not get sick enough to +be seriously uncomfortable. They were not likely to fool away this high +promise for lack of effort. No, they practised cautiously, after +supper, with right fair success, and so they spent a jubilant evening. +They were prouder and happier in their new acquirement than they would +have been in the scalping and skinning of the Six Nations. We will +leave them to smoke and chatter and brag, since we have no further use +for them at present. + + + +CHAPTER XVII + +BUT there was no hilarity in the little town that same tranquil +Saturday afternoon. The Harpers, and Aunt Polly's family, were being +put into mourning, with great grief and many tears. An unusual quiet +possessed the village, although it was ordinarily quiet enough, in all +conscience. The villagers conducted their concerns with an absent air, +and talked little; but they sighed often. The Saturday holiday seemed a +burden to the children. They had no heart in their sports, and +gradually gave them up. + +In the afternoon Becky Thatcher found herself moping about the +deserted schoolhouse yard, and feeling very melancholy. But she found +nothing there to comfort her. She soliloquized: + +"Oh, if I only had a brass andiron-knob again! But I haven't got +anything now to remember him by." And she choked back a little sob. + +Presently she stopped, and said to herself: + +"It was right here. Oh, if it was to do over again, I wouldn't say +that--I wouldn't say it for the whole world. But he's gone now; I'll +never, never, never see him any more." + +This thought broke her down, and she wandered away, with tears rolling +down her cheeks. Then quite a group of boys and girls--playmates of +Tom's and Joe's--came by, and stood looking over the paling fence and +talking in reverent tones of how Tom did so-and-so the last time they +saw him, and how Joe said this and that small trifle (pregnant with +awful prophecy, as they could easily see now!)--and each speaker +pointed out the exact spot where the lost lads stood at the time, and +then added something like "and I was a-standing just so--just as I am +now, and as if you was him--I was as close as that--and he smiled, just +this way--and then something seemed to go all over me, like--awful, you +know--and I never thought what it meant, of course, but I can see now!" + +Then there was a dispute about who saw the dead boys last in life, and +many claimed that dismal distinction, and offered evidences, more or +less tampered with by the witness; and when it was ultimately decided +who DID see the departed last, and exchanged the last words with them, +the lucky parties took upon themselves a sort of sacred importance, and +were gaped at and envied by all the rest. One poor chap, who had no +other grandeur to offer, said with tolerably manifest pride in the +remembrance: + +"Well, Tom Sawyer he licked me once." + +But that bid for glory was a failure. Most of the boys could say that, +and so that cheapened the distinction too much. The group loitered +away, still recalling memories of the lost heroes, in awed voices. + +When the Sunday-school hour was finished, the next morning, the bell +began to toll, instead of ringing in the usual way. It was a very still +Sabbath, and the mournful sound seemed in keeping with the musing hush +that lay upon nature. The villagers began to gather, loitering a moment +in the vestibule to converse in whispers about the sad event. But there +was no whispering in the house; only the funereal rustling of dresses +as the women gathered to their seats disturbed the silence there. None +could remember when the little church had been so full before. There +was finally a waiting pause, an expectant dumbness, and then Aunt Polly +entered, followed by Sid and Mary, and they by the Harper family, all +in deep black, and the whole congregation, the old minister as well, +rose reverently and stood until the mourners were seated in the front +pew. There was another communing silence, broken at intervals by +muffled sobs, and then the minister spread his hands abroad and prayed. +A moving hymn was sung, and the text followed: "I am the Resurrection +and the Life." + +As the service proceeded, the clergyman drew such pictures of the +graces, the winning ways, and the rare promise of the lost lads that +every soul there, thinking he recognized these pictures, felt a pang in +remembering that he had persistently blinded himself to them always +before, and had as persistently seen only faults and flaws in the poor +boys. The minister related many a touching incident in the lives of the +departed, too, which illustrated their sweet, generous natures, and the +people could easily see, now, how noble and beautiful those episodes +were, and remembered with grief that at the time they occurred they had +seemed rank rascalities, well deserving of the cowhide. The +congregation became more and more moved, as the pathetic tale went on, +till at last the whole company broke down and joined the weeping +mourners in a chorus of anguished sobs, the preacher himself giving way +to his feelings, and crying in the pulpit. + +There was a rustle in the gallery, which nobody noticed; a moment +later the church door creaked; the minister raised his streaming eyes +above his handkerchief, and stood transfixed! First one and then +another pair of eyes followed the minister's, and then almost with one +impulse the congregation rose and stared while the three dead boys came +marching up the aisle, Tom in the lead, Joe next, and Huck, a ruin of +drooping rags, sneaking sheepishly in the rear! They had been hid in +the unused gallery listening to their own funeral sermon! + +Aunt Polly, Mary, and the Harpers threw themselves upon their restored +ones, smothered them with kisses and poured out thanksgivings, while +poor Huck stood abashed and uncomfortable, not knowing exactly what to +do or where to hide from so many unwelcoming eyes. He wavered, and +started to slink away, but Tom seized him and said: + +"Aunt Polly, it ain't fair. Somebody's got to be glad to see Huck." + +"And so they shall. I'm glad to see him, poor motherless thing!" And +the loving attentions Aunt Polly lavished upon him were the one thing +capable of making him more uncomfortable than he was before. + +Suddenly the minister shouted at the top of his voice: "Praise God +from whom all blessings flow--SING!--and put your hearts in it!" + +And they did. Old Hundred swelled up with a triumphant burst, and +while it shook the rafters Tom Sawyer the Pirate looked around upon the +envying juveniles about him and confessed in his heart that this was +the proudest moment of his life. + +As the "sold" congregation trooped out they said they would almost be +willing to be made ridiculous again to hear Old Hundred sung like that +once more. + +Tom got more cuffs and kisses that day--according to Aunt Polly's +varying moods--than he had earned before in a year; and he hardly knew +which expressed the most gratefulness to God and affection for himself. + + + +CHAPTER XVIII + +THAT was Tom's great secret--the scheme to return home with his +brother pirates and attend their own funerals. They had paddled over to +the Missouri shore on a log, at dusk on Saturday, landing five or six +miles below the village; they had slept in the woods at the edge of the +town till nearly daylight, and had then crept through back lanes and +alleys and finished their sleep in the gallery of the church among a +chaos of invalided benches. + +At breakfast, Monday morning, Aunt Polly and Mary were very loving to +Tom, and very attentive to his wants. There was an unusual amount of +talk. In the course of it Aunt Polly said: + +"Well, I don't say it wasn't a fine joke, Tom, to keep everybody +suffering 'most a week so you boys had a good time, but it is a pity +you could be so hard-hearted as to let me suffer so. If you could come +over on a log to go to your funeral, you could have come over and give +me a hint some way that you warn't dead, but only run off." + +"Yes, you could have done that, Tom," said Mary; "and I believe you +would if you had thought of it." + +"Would you, Tom?" said Aunt Polly, her face lighting wistfully. "Say, +now, would you, if you'd thought of it?" + +"I--well, I don't know. 'Twould 'a' spoiled everything." + +"Tom, I hoped you loved me that much," said Aunt Polly, with a grieved +tone that discomforted the boy. "It would have been something if you'd +cared enough to THINK of it, even if you didn't DO it." + +"Now, auntie, that ain't any harm," pleaded Mary; "it's only Tom's +giddy way--he is always in such a rush that he never thinks of +anything." + +"More's the pity. Sid would have thought. And Sid would have come and +DONE it, too. Tom, you'll look back, some day, when it's too late, and +wish you'd cared a little more for me when it would have cost you so +little." + +"Now, auntie, you know I do care for you," said Tom. + +"I'd know it better if you acted more like it." + +"I wish now I'd thought," said Tom, with a repentant tone; "but I +dreamt about you, anyway. That's something, ain't it?" + +"It ain't much--a cat does that much--but it's better than nothing. +What did you dream?" + +"Why, Wednesday night I dreamt that you was sitting over there by the +bed, and Sid was sitting by the woodbox, and Mary next to him." + +"Well, so we did. So we always do. I'm glad your dreams could take +even that much trouble about us." + +"And I dreamt that Joe Harper's mother was here." + +"Why, she was here! Did you dream any more?" + +"Oh, lots. But it's so dim, now." + +"Well, try to recollect--can't you?" + +"Somehow it seems to me that the wind--the wind blowed the--the--" + +"Try harder, Tom! The wind did blow something. Come!" + +Tom pressed his fingers on his forehead an anxious minute, and then +said: + +"I've got it now! I've got it now! It blowed the candle!" + +"Mercy on us! Go on, Tom--go on!" + +"And it seems to me that you said, 'Why, I believe that that door--'" + +"Go ON, Tom!" + +"Just let me study a moment--just a moment. Oh, yes--you said you +believed the door was open." + +"As I'm sitting here, I did! Didn't I, Mary! Go on!" + +"And then--and then--well I won't be certain, but it seems like as if +you made Sid go and--and--" + +"Well? Well? What did I make him do, Tom? What did I make him do?" + +"You made him--you--Oh, you made him shut it." + +"Well, for the land's sake! I never heard the beat of that in all my +days! Don't tell ME there ain't anything in dreams, any more. Sereny +Harper shall know of this before I'm an hour older. I'd like to see her +get around THIS with her rubbage 'bout superstition. Go on, Tom!" + +"Oh, it's all getting just as bright as day, now. Next you said I +warn't BAD, only mischeevous and harum-scarum, and not any more +responsible than--than--I think it was a colt, or something." + +"And so it was! Well, goodness gracious! Go on, Tom!" + +"And then you began to cry." + +"So I did. So I did. Not the first time, neither. And then--" + +"Then Mrs. Harper she began to cry, and said Joe was just the same, +and she wished she hadn't whipped him for taking cream when she'd +throwed it out her own self--" + +"Tom! The sperrit was upon you! You was a prophesying--that's what you +was doing! Land alive, go on, Tom!" + +"Then Sid he said--he said--" + +"I don't think I said anything," said Sid. + +"Yes you did, Sid," said Mary. + +"Shut your heads and let Tom go on! What did he say, Tom?" + +"He said--I THINK he said he hoped I was better off where I was gone +to, but if I'd been better sometimes--" + +"THERE, d'you hear that! It was his very words!" + +"And you shut him up sharp." + +"I lay I did! There must 'a' been an angel there. There WAS an angel +there, somewheres!" + +"And Mrs. Harper told about Joe scaring her with a firecracker, and +you told about Peter and the Painkiller--" + +"Just as true as I live!" + +"And then there was a whole lot of talk 'bout dragging the river for +us, and 'bout having the funeral Sunday, and then you and old Miss +Harper hugged and cried, and she went." + +"It happened just so! It happened just so, as sure as I'm a-sitting in +these very tracks. Tom, you couldn't told it more like if you'd 'a' +seen it! And then what? Go on, Tom!" + +"Then I thought you prayed for me--and I could see you and hear every +word you said. And you went to bed, and I was so sorry that I took and +wrote on a piece of sycamore bark, 'We ain't dead--we are only off +being pirates,' and put it on the table by the candle; and then you +looked so good, laying there asleep, that I thought I went and leaned +over and kissed you on the lips." + +"Did you, Tom, DID you! I just forgive you everything for that!" And +she seized the boy in a crushing embrace that made him feel like the +guiltiest of villains. + +"It was very kind, even though it was only a--dream," Sid soliloquized +just audibly. + +"Shut up, Sid! A body does just the same in a dream as he'd do if he +was awake. Here's a big Milum apple I've been saving for you, Tom, if +you was ever found again--now go 'long to school. I'm thankful to the +good God and Father of us all I've got you back, that's long-suffering +and merciful to them that believe on Him and keep His word, though +goodness knows I'm unworthy of it, but if only the worthy ones got His +blessings and had His hand to help them over the rough places, there's +few enough would smile here or ever enter into His rest when the long +night comes. Go 'long Sid, Mary, Tom--take yourselves off--you've +hendered me long enough." + +The children left for school, and the old lady to call on Mrs. Harper +and vanquish her realism with Tom's marvellous dream. Sid had better +judgment than to utter the thought that was in his mind as he left the +house. It was this: "Pretty thin--as long a dream as that, without any +mistakes in it!" + +What a hero Tom was become, now! He did not go skipping and prancing, +but moved with a dignified swagger as became a pirate who felt that the +public eye was on him. And indeed it was; he tried not to seem to see +the looks or hear the remarks as he passed along, but they were food +and drink to him. Smaller boys than himself flocked at his heels, as +proud to be seen with him, and tolerated by him, as if he had been the +drummer at the head of a procession or the elephant leading a menagerie +into town. Boys of his own size pretended not to know he had been away +at all; but they were consuming with envy, nevertheless. They would +have given anything to have that swarthy suntanned skin of his, and his +glittering notoriety; and Tom would not have parted with either for a +circus. + +At school the children made so much of him and of Joe, and delivered +such eloquent admiration from their eyes, that the two heroes were not +long in becoming insufferably "stuck-up." They began to tell their +adventures to hungry listeners--but they only began; it was not a thing +likely to have an end, with imaginations like theirs to furnish +material. And finally, when they got out their pipes and went serenely +puffing around, the very summit of glory was reached. + +Tom decided that he could be independent of Becky Thatcher now. Glory +was sufficient. He would live for glory. Now that he was distinguished, +maybe she would be wanting to "make up." Well, let her--she should see +that he could be as indifferent as some other people. Presently she +arrived. Tom pretended not to see her. He moved away and joined a group +of boys and girls and began to talk. Soon he observed that she was +tripping gayly back and forth with flushed face and dancing eyes, +pretending to be busy chasing schoolmates, and screaming with laughter +when she made a capture; but he noticed that she always made her +captures in his vicinity, and that she seemed to cast a conscious eye +in his direction at such times, too. It gratified all the vicious +vanity that was in him; and so, instead of winning him, it only "set +him up" the more and made him the more diligent to avoid betraying that +he knew she was about. Presently she gave over skylarking, and moved +irresolutely about, sighing once or twice and glancing furtively and +wistfully toward Tom. Then she observed that now Tom was talking more +particularly to Amy Lawrence than to any one else. She felt a sharp +pang and grew disturbed and uneasy at once. She tried to go away, but +her feet were treacherous, and carried her to the group instead. She +said to a girl almost at Tom's elbow--with sham vivacity: + +"Why, Mary Austin! you bad girl, why didn't you come to Sunday-school?" + +"I did come--didn't you see me?" + +"Why, no! Did you? Where did you sit?" + +"I was in Miss Peters' class, where I always go. I saw YOU." + +"Did you? Why, it's funny I didn't see you. I wanted to tell you about +the picnic." + +"Oh, that's jolly. Who's going to give it?" + +"My ma's going to let me have one." + +"Oh, goody; I hope she'll let ME come." + +"Well, she will. The picnic's for me. She'll let anybody come that I +want, and I want you." + +"That's ever so nice. When is it going to be?" + +"By and by. Maybe about vacation." + +"Oh, won't it be fun! You going to have all the girls and boys?" + +"Yes, every one that's friends to me--or wants to be"; and she glanced +ever so furtively at Tom, but he talked right along to Amy Lawrence +about the terrible storm on the island, and how the lightning tore the +great sycamore tree "all to flinders" while he was "standing within +three feet of it." + +"Oh, may I come?" said Grace Miller. + +"Yes." + +"And me?" said Sally Rogers. + +"Yes." + +"And me, too?" said Susy Harper. "And Joe?" + +"Yes." + +And so on, with clapping of joyful hands till all the group had begged +for invitations but Tom and Amy. Then Tom turned coolly away, still +talking, and took Amy with him. Becky's lips trembled and the tears +came to her eyes; she hid these signs with a forced gayety and went on +chattering, but the life had gone out of the picnic, now, and out of +everything else; she got away as soon as she could and hid herself and +had what her sex call "a good cry." Then she sat moody, with wounded +pride, till the bell rang. She roused up, now, with a vindictive cast +in her eye, and gave her plaited tails a shake and said she knew what +SHE'D do. + +At recess Tom continued his flirtation with Amy with jubilant +self-satisfaction. And he kept drifting about to find Becky and lacerate +her with the performance. At last he spied her, but there was a sudden +falling of his mercury. She was sitting cosily on a little bench behind +the schoolhouse looking at a picture-book with Alfred Temple--and so +absorbed were they, and their heads so close together over the book, +that they did not seem to be conscious of anything in the world besides. +Jealousy ran red-hot through Tom's veins. He began to hate himself for +throwing away the chance Becky had offered for a reconciliation. He +called himself a fool, and all the hard names he could think of. He +wanted to cry with vexation. Amy chatted happily along, as they walked, +for her heart was singing, but Tom's tongue had lost its function. He +did not hear what Amy was saying, and whenever she paused expectantly he +could only stammer an awkward assent, which was as often misplaced as +otherwise. He kept drifting to the rear of the schoolhouse, again and +again, to sear his eyeballs with the hateful spectacle there. He could +not help it. And it maddened him to see, as he thought he saw, that +Becky Thatcher never once suspected that he was even in the land of the +living. But she did see, nevertheless; and she knew she was winning her +fight, too, and was glad to see him suffer as she had suffered. + +Amy's happy prattle became intolerable. Tom hinted at things he had to +attend to; things that must be done; and time was fleeting. But in +vain--the girl chirped on. Tom thought, "Oh, hang her, ain't I ever +going to get rid of her?" At last he must be attending to those +things--and she said artlessly that she would be "around" when school +let out. And he hastened away, hating her for it. + +"Any other boy!" Tom thought, grating his teeth. "Any boy in the whole +town but that Saint Louis smarty that thinks he dresses so fine and is +aristocracy! Oh, all right, I licked you the first day you ever saw +this town, mister, and I'll lick you again! You just wait till I catch +you out! I'll just take and--" + +And he went through the motions of thrashing an imaginary boy +--pummelling the air, and kicking and gouging. "Oh, you do, do you? You +holler 'nough, do you? Now, then, let that learn you!" And so the +imaginary flogging was finished to his satisfaction. + +Tom fled home at noon. His conscience could not endure any more of +Amy's grateful happiness, and his jealousy could bear no more of the +other distress. Becky resumed her picture inspections with Alfred, but +as the minutes dragged along and no Tom came to suffer, her triumph +began to cloud and she lost interest; gravity and absent-mindedness +followed, and then melancholy; two or three times she pricked up her +ear at a footstep, but it was a false hope; no Tom came. At last she +grew entirely miserable and wished she hadn't carried it so far. When +poor Alfred, seeing that he was losing her, he did not know how, kept +exclaiming: "Oh, here's a jolly one! look at this!" she lost patience +at last, and said, "Oh, don't bother me! I don't care for them!" and +burst into tears, and got up and walked away. + +Alfred dropped alongside and was going to try to comfort her, but she +said: + +"Go away and leave me alone, can't you! I hate you!" + +So the boy halted, wondering what he could have done--for she had said +she would look at pictures all through the nooning--and she walked on, +crying. Then Alfred went musing into the deserted schoolhouse. He was +humiliated and angry. He easily guessed his way to the truth--the girl +had simply made a convenience of him to vent her spite upon Tom Sawyer. +He was far from hating Tom the less when this thought occurred to him. +He wished there was some way to get that boy into trouble without much +risk to himself. Tom's spelling-book fell under his eye. Here was his +opportunity. He gratefully opened to the lesson for the afternoon and +poured ink upon the page. + +Becky, glancing in at a window behind him at the moment, saw the act, +and moved on, without discovering herself. She started homeward, now, +intending to find Tom and tell him; Tom would be thankful and their +troubles would be healed. Before she was half way home, however, she +had changed her mind. The thought of Tom's treatment of her when she +was talking about her picnic came scorching back and filled her with +shame. She resolved to let him get whipped on the damaged +spelling-book's account, and to hate him forever, into the bargain. + + + +CHAPTER XIX + +TOM arrived at home in a dreary mood, and the first thing his aunt +said to him showed him that he had brought his sorrows to an +unpromising market: + +"Tom, I've a notion to skin you alive!" + +"Auntie, what have I done?" + +"Well, you've done enough. Here I go over to Sereny Harper, like an +old softy, expecting I'm going to make her believe all that rubbage +about that dream, when lo and behold you she'd found out from Joe that +you was over here and heard all the talk we had that night. Tom, I +don't know what is to become of a boy that will act like that. It makes +me feel so bad to think you could let me go to Sereny Harper and make +such a fool of myself and never say a word." + +This was a new aspect of the thing. His smartness of the morning had +seemed to Tom a good joke before, and very ingenious. It merely looked +mean and shabby now. He hung his head and could not think of anything +to say for a moment. Then he said: + +"Auntie, I wish I hadn't done it--but I didn't think." + +"Oh, child, you never think. You never think of anything but your own +selfishness. You could think to come all the way over here from +Jackson's Island in the night to laugh at our troubles, and you could +think to fool me with a lie about a dream; but you couldn't ever think +to pity us and save us from sorrow." + +"Auntie, I know now it was mean, but I didn't mean to be mean. I +didn't, honest. And besides, I didn't come over here to laugh at you +that night." + +"What did you come for, then?" + +"It was to tell you not to be uneasy about us, because we hadn't got +drownded." + +"Tom, Tom, I would be the thankfullest soul in this world if I could +believe you ever had as good a thought as that, but you know you never +did--and I know it, Tom." + +"Indeed and 'deed I did, auntie--I wish I may never stir if I didn't." + +"Oh, Tom, don't lie--don't do it. It only makes things a hundred times +worse." + +"It ain't a lie, auntie; it's the truth. I wanted to keep you from +grieving--that was all that made me come." + +"I'd give the whole world to believe that--it would cover up a power +of sins, Tom. I'd 'most be glad you'd run off and acted so bad. But it +ain't reasonable; because, why didn't you tell me, child?" + +"Why, you see, when you got to talking about the funeral, I just got +all full of the idea of our coming and hiding in the church, and I +couldn't somehow bear to spoil it. So I just put the bark back in my +pocket and kept mum." + +"What bark?" + +"The bark I had wrote on to tell you we'd gone pirating. I wish, now, +you'd waked up when I kissed you--I do, honest." + +The hard lines in his aunt's face relaxed and a sudden tenderness +dawned in her eyes. + +"DID you kiss me, Tom?" + +"Why, yes, I did." + +"Are you sure you did, Tom?" + +"Why, yes, I did, auntie--certain sure." + +"What did you kiss me for, Tom?" + +"Because I loved you so, and you laid there moaning and I was so sorry." + +The words sounded like truth. The old lady could not hide a tremor in +her voice when she said: + +"Kiss me again, Tom!--and be off with you to school, now, and don't +bother me any more." + +The moment he was gone, she ran to a closet and got out the ruin of a +jacket which Tom had gone pirating in. Then she stopped, with it in her +hand, and said to herself: + +"No, I don't dare. Poor boy, I reckon he's lied about it--but it's a +blessed, blessed lie, there's such a comfort come from it. I hope the +Lord--I KNOW the Lord will forgive him, because it was such +goodheartedness in him to tell it. But I don't want to find out it's a +lie. I won't look." + +She put the jacket away, and stood by musing a minute. Twice she put +out her hand to take the garment again, and twice she refrained. Once +more she ventured, and this time she fortified herself with the +thought: "It's a good lie--it's a good lie--I won't let it grieve me." +So she sought the jacket pocket. A moment later she was reading Tom's +piece of bark through flowing tears and saying: "I could forgive the +boy, now, if he'd committed a million sins!" + + + +CHAPTER XX + +THERE was something about Aunt Polly's manner, when she kissed Tom, +that swept away his low spirits and made him lighthearted and happy +again. He started to school and had the luck of coming upon Becky +Thatcher at the head of Meadow Lane. His mood always determined his +manner. Without a moment's hesitation he ran to her and said: + +"I acted mighty mean to-day, Becky, and I'm so sorry. I won't ever, +ever do that way again, as long as ever I live--please make up, won't +you?" + +The girl stopped and looked him scornfully in the face: + +"I'll thank you to keep yourself TO yourself, Mr. Thomas Sawyer. I'll +never speak to you again." + +She tossed her head and passed on. Tom was so stunned that he had not +even presence of mind enough to say "Who cares, Miss Smarty?" until the +right time to say it had gone by. So he said nothing. But he was in a +fine rage, nevertheless. He moped into the schoolyard wishing she were +a boy, and imagining how he would trounce her if she were. He presently +encountered her and delivered a stinging remark as he passed. She +hurled one in return, and the angry breach was complete. It seemed to +Becky, in her hot resentment, that she could hardly wait for school to +"take in," she was so impatient to see Tom flogged for the injured +spelling-book. If she had had any lingering notion of exposing Alfred +Temple, Tom's offensive fling had driven it entirely away. + +Poor girl, she did not know how fast she was nearing trouble herself. +The master, Mr. Dobbins, had reached middle age with an unsatisfied +ambition. The darling of his desires was, to be a doctor, but poverty +had decreed that he should be nothing higher than a village +schoolmaster. Every day he took a mysterious book out of his desk and +absorbed himself in it at times when no classes were reciting. He kept +that book under lock and key. There was not an urchin in school but was +perishing to have a glimpse of it, but the chance never came. Every boy +and girl had a theory about the nature of that book; but no two +theories were alike, and there was no way of getting at the facts in +the case. Now, as Becky was passing by the desk, which stood near the +door, she noticed that the key was in the lock! It was a precious +moment. She glanced around; found herself alone, and the next instant +she had the book in her hands. The title-page--Professor Somebody's +ANATOMY--carried no information to her mind; so she began to turn the +leaves. She came at once upon a handsomely engraved and colored +frontispiece--a human figure, stark naked. At that moment a shadow fell +on the page and Tom Sawyer stepped in at the door and caught a glimpse +of the picture. Becky snatched at the book to close it, and had the +hard luck to tear the pictured page half down the middle. She thrust +the volume into the desk, turned the key, and burst out crying with +shame and vexation. + +"Tom Sawyer, you are just as mean as you can be, to sneak up on a +person and look at what they're looking at." + +"How could I know you was looking at anything?" + +"You ought to be ashamed of yourself, Tom Sawyer; you know you're +going to tell on me, and oh, what shall I do, what shall I do! I'll be +whipped, and I never was whipped in school." + +Then she stamped her little foot and said: + +"BE so mean if you want to! I know something that's going to happen. +You just wait and you'll see! Hateful, hateful, hateful!"--and she +flung out of the house with a new explosion of crying. + +Tom stood still, rather flustered by this onslaught. Presently he said +to himself: + +"What a curious kind of a fool a girl is! Never been licked in school! +Shucks! What's a licking! That's just like a girl--they're so +thin-skinned and chicken-hearted. Well, of course I ain't going to tell +old Dobbins on this little fool, because there's other ways of getting +even on her, that ain't so mean; but what of it? Old Dobbins will ask +who it was tore his book. Nobody'll answer. Then he'll do just the way +he always does--ask first one and then t'other, and when he comes to the +right girl he'll know it, without any telling. Girls' faces always tell +on them. They ain't got any backbone. She'll get licked. Well, it's a +kind of a tight place for Becky Thatcher, because there ain't any way +out of it." Tom conned the thing a moment longer, and then added: "All +right, though; she'd like to see me in just such a fix--let her sweat it +out!" + +Tom joined the mob of skylarking scholars outside. In a few moments +the master arrived and school "took in." Tom did not feel a strong +interest in his studies. Every time he stole a glance at the girls' +side of the room Becky's face troubled him. Considering all things, he +did not want to pity her, and yet it was all he could do to help it. He +could get up no exultation that was really worthy the name. Presently +the spelling-book discovery was made, and Tom's mind was entirely full +of his own matters for a while after that. Becky roused up from her +lethargy of distress and showed good interest in the proceedings. She +did not expect that Tom could get out of his trouble by denying that he +spilt the ink on the book himself; and she was right. The denial only +seemed to make the thing worse for Tom. Becky supposed she would be +glad of that, and she tried to believe she was glad of it, but she +found she was not certain. When the worst came to the worst, she had an +impulse to get up and tell on Alfred Temple, but she made an effort and +forced herself to keep still--because, said she to herself, "he'll tell +about me tearing the picture sure. I wouldn't say a word, not to save +his life!" + +Tom took his whipping and went back to his seat not at all +broken-hearted, for he thought it was possible that he had unknowingly +upset the ink on the spelling-book himself, in some skylarking bout--he +had denied it for form's sake and because it was custom, and had stuck +to the denial from principle. + +A whole hour drifted by, the master sat nodding in his throne, the air +was drowsy with the hum of study. By and by, Mr. Dobbins straightened +himself up, yawned, then unlocked his desk, and reached for his book, +but seemed undecided whether to take it out or leave it. Most of the +pupils glanced up languidly, but there were two among them that watched +his movements with intent eyes. Mr. Dobbins fingered his book absently +for a while, then took it out and settled himself in his chair to read! +Tom shot a glance at Becky. He had seen a hunted and helpless rabbit +look as she did, with a gun levelled at its head. Instantly he forgot +his quarrel with her. Quick--something must be done! done in a flash, +too! But the very imminence of the emergency paralyzed his invention. +Good!--he had an inspiration! He would run and snatch the book, spring +through the door and fly. But his resolution shook for one little +instant, and the chance was lost--the master opened the volume. If Tom +only had the wasted opportunity back again! Too late. There was no help +for Becky now, he said. The next moment the master faced the school. +Every eye sank under his gaze. There was that in it which smote even +the innocent with fear. There was silence while one might count ten +--the master was gathering his wrath. Then he spoke: "Who tore this book?" + +There was not a sound. One could have heard a pin drop. The stillness +continued; the master searched face after face for signs of guilt. + +"Benjamin Rogers, did you tear this book?" + +A denial. Another pause. + +"Joseph Harper, did you?" + +Another denial. Tom's uneasiness grew more and more intense under the +slow torture of these proceedings. The master scanned the ranks of +boys--considered a while, then turned to the girls: + +"Amy Lawrence?" + +A shake of the head. + +"Gracie Miller?" + +The same sign. + +"Susan Harper, did you do this?" + +Another negative. The next girl was Becky Thatcher. Tom was trembling +from head to foot with excitement and a sense of the hopelessness of +the situation. + +"Rebecca Thatcher" [Tom glanced at her face--it was white with terror] +--"did you tear--no, look me in the face" [her hands rose in appeal] +--"did you tear this book?" + +A thought shot like lightning through Tom's brain. He sprang to his +feet and shouted--"I done it!" + +The school stared in perplexity at this incredible folly. Tom stood a +moment, to gather his dismembered faculties; and when he stepped +forward to go to his punishment the surprise, the gratitude, the +adoration that shone upon him out of poor Becky's eyes seemed pay +enough for a hundred floggings. Inspired by the splendor of his own +act, he took without an outcry the most merciless flaying that even Mr. +Dobbins had ever administered; and also received with indifference the +added cruelty of a command to remain two hours after school should be +dismissed--for he knew who would wait for him outside till his +captivity was done, and not count the tedious time as loss, either. + +Tom went to bed that night planning vengeance against Alfred Temple; +for with shame and repentance Becky had told him all, not forgetting +her own treachery; but even the longing for vengeance had to give way, +soon, to pleasanter musings, and he fell asleep at last with Becky's +latest words lingering dreamily in his ear-- + +"Tom, how COULD you be so noble!" + + + +CHAPTER XXI + +VACATION was approaching. The schoolmaster, always severe, grew +severer and more exacting than ever, for he wanted the school to make a +good showing on "Examination" day. His rod and his ferule were seldom +idle now--at least among the smaller pupils. Only the biggest boys, and +young ladies of eighteen and twenty, escaped lashing. Mr. Dobbins' +lashings were very vigorous ones, too; for although he carried, under +his wig, a perfectly bald and shiny head, he had only reached middle +age, and there was no sign of feebleness in his muscle. As the great +day approached, all the tyranny that was in him came to the surface; he +seemed to take a vindictive pleasure in punishing the least +shortcomings. The consequence was, that the smaller boys spent their +days in terror and suffering and their nights in plotting revenge. They +threw away no opportunity to do the master a mischief. But he kept +ahead all the time. The retribution that followed every vengeful +success was so sweeping and majestic that the boys always retired from +the field badly worsted. At last they conspired together and hit upon a +plan that promised a dazzling victory. They swore in the sign-painter's +boy, told him the scheme, and asked his help. He had his own reasons +for being delighted, for the master boarded in his father's family and +had given the boy ample cause to hate him. The master's wife would go +on a visit to the country in a few days, and there would be nothing to +interfere with the plan; the master always prepared himself for great +occasions by getting pretty well fuddled, and the sign-painter's boy +said that when the dominie had reached the proper condition on +Examination Evening he would "manage the thing" while he napped in his +chair; then he would have him awakened at the right time and hurried +away to school. + +In the fulness of time the interesting occasion arrived. At eight in +the evening the schoolhouse was brilliantly lighted, and adorned with +wreaths and festoons of foliage and flowers. The master sat throned in +his great chair upon a raised platform, with his blackboard behind him. +He was looking tolerably mellow. Three rows of benches on each side and +six rows in front of him were occupied by the dignitaries of the town +and by the parents of the pupils. To his left, back of the rows of +citizens, was a spacious temporary platform upon which were seated the +scholars who were to take part in the exercises of the evening; rows of +small boys, washed and dressed to an intolerable state of discomfort; +rows of gawky big boys; snowbanks of girls and young ladies clad in +lawn and muslin and conspicuously conscious of their bare arms, their +grandmothers' ancient trinkets, their bits of pink and blue ribbon and +the flowers in their hair. All the rest of the house was filled with +non-participating scholars. + +The exercises began. A very little boy stood up and sheepishly +recited, "You'd scarce expect one of my age to speak in public on the +stage," etc.--accompanying himself with the painfully exact and +spasmodic gestures which a machine might have used--supposing the +machine to be a trifle out of order. But he got through safely, though +cruelly scared, and got a fine round of applause when he made his +manufactured bow and retired. + +A little shamefaced girl lisped, "Mary had a little lamb," etc., +performed a compassion-inspiring curtsy, got her meed of applause, and +sat down flushed and happy. + +Tom Sawyer stepped forward with conceited confidence and soared into +the unquenchable and indestructible "Give me liberty or give me death" +speech, with fine fury and frantic gesticulation, and broke down in the +middle of it. A ghastly stage-fright seized him, his legs quaked under +him and he was like to choke. True, he had the manifest sympathy of the +house but he had the house's silence, too, which was even worse than +its sympathy. The master frowned, and this completed the disaster. Tom +struggled awhile and then retired, utterly defeated. There was a weak +attempt at applause, but it died early. + +"The Boy Stood on the Burning Deck" followed; also "The Assyrian Came +Down," and other declamatory gems. Then there were reading exercises, +and a spelling fight. The meagre Latin class recited with honor. The +prime feature of the evening was in order, now--original "compositions" +by the young ladies. Each in her turn stepped forward to the edge of +the platform, cleared her throat, held up her manuscript (tied with +dainty ribbon), and proceeded to read, with labored attention to +"expression" and punctuation. The themes were the same that had been +illuminated upon similar occasions by their mothers before them, their +grandmothers, and doubtless all their ancestors in the female line +clear back to the Crusades. "Friendship" was one; "Memories of Other +Days"; "Religion in History"; "Dream Land"; "The Advantages of +Culture"; "Forms of Political Government Compared and Contrasted"; +"Melancholy"; "Filial Love"; "Heart Longings," etc., etc. + +A prevalent feature in these compositions was a nursed and petted +melancholy; another was a wasteful and opulent gush of "fine language"; +another was a tendency to lug in by the ears particularly prized words +and phrases until they were worn entirely out; and a peculiarity that +conspicuously marked and marred them was the inveterate and intolerable +sermon that wagged its crippled tail at the end of each and every one +of them. No matter what the subject might be, a brain-racking effort +was made to squirm it into some aspect or other that the moral and +religious mind could contemplate with edification. The glaring +insincerity of these sermons was not sufficient to compass the +banishment of the fashion from the schools, and it is not sufficient +to-day; it never will be sufficient while the world stands, perhaps. +There is no school in all our land where the young ladies do not feel +obliged to close their compositions with a sermon; and you will find +that the sermon of the most frivolous and the least religious girl in +the school is always the longest and the most relentlessly pious. But +enough of this. Homely truth is unpalatable. + +Let us return to the "Examination." The first composition that was +read was one entitled "Is this, then, Life?" Perhaps the reader can +endure an extract from it: + + "In the common walks of life, with what delightful + emotions does the youthful mind look forward to some + anticipated scene of festivity! Imagination is busy + sketching rose-tinted pictures of joy. In fancy, the + voluptuous votary of fashion sees herself amid the + festive throng, 'the observed of all observers.' Her + graceful form, arrayed in snowy robes, is whirling + through the mazes of the joyous dance; her eye is + brightest, her step is lightest in the gay assembly. + + "In such delicious fancies time quickly glides by, + and the welcome hour arrives for her entrance into + the Elysian world, of which she has had such bright + dreams. How fairy-like does everything appear to + her enchanted vision! Each new scene is more charming + than the last. But after a while she finds that + beneath this goodly exterior, all is vanity, the + flattery which once charmed her soul, now grates + harshly upon her ear; the ball-room has lost its + charms; and with wasted health and imbittered heart, + she turns away with the conviction that earthly + pleasures cannot satisfy the longings of the soul!" + +And so forth and so on. There was a buzz of gratification from time to +time during the reading, accompanied by whispered ejaculations of "How +sweet!" "How eloquent!" "So true!" etc., and after the thing had closed +with a peculiarly afflicting sermon the applause was enthusiastic. + +Then arose a slim, melancholy girl, whose face had the "interesting" +paleness that comes of pills and indigestion, and read a "poem." Two +stanzas of it will do: + + "A MISSOURI MAIDEN'S FAREWELL TO ALABAMA + + "Alabama, good-bye! I love thee well! + But yet for a while do I leave thee now! + Sad, yes, sad thoughts of thee my heart doth swell, + And burning recollections throng my brow! + For I have wandered through thy flowery woods; + Have roamed and read near Tallapoosa's stream; + Have listened to Tallassee's warring floods, + And wooed on Coosa's side Aurora's beam. + + "Yet shame I not to bear an o'er-full heart, + Nor blush to turn behind my tearful eyes; + 'Tis from no stranger land I now must part, + 'Tis to no strangers left I yield these sighs. + Welcome and home were mine within this State, + Whose vales I leave--whose spires fade fast from me + And cold must be mine eyes, and heart, and tete, + When, dear Alabama! they turn cold on thee!" + +There were very few there who knew what "tete" meant, but the poem was +very satisfactory, nevertheless. + +Next appeared a dark-complexioned, black-eyed, black-haired young +lady, who paused an impressive moment, assumed a tragic expression, and +began to read in a measured, solemn tone: + + "A VISION + + "Dark and tempestuous was night. Around the + throne on high not a single star quivered; but + the deep intonations of the heavy thunder + constantly vibrated upon the ear; whilst the + terrific lightning revelled in angry mood + through the cloudy chambers of heaven, seeming + to scorn the power exerted over its terror by + the illustrious Franklin! Even the boisterous + winds unanimously came forth from their mystic + homes, and blustered about as if to enhance by + their aid the wildness of the scene. + + "At such a time, so dark, so dreary, for human + sympathy my very spirit sighed; but instead thereof, + + "'My dearest friend, my counsellor, my comforter + and guide--My joy in grief, my second bliss + in joy,' came to my side. She moved like one of + those bright beings pictured in the sunny walks + of fancy's Eden by the romantic and young, a + queen of beauty unadorned save by her own + transcendent loveliness. So soft was her step, it + failed to make even a sound, and but for the + magical thrill imparted by her genial touch, as + other unobtrusive beauties, she would have glided + away un-perceived--unsought. A strange sadness + rested upon her features, like icy tears upon + the robe of December, as she pointed to the + contending elements without, and bade me contemplate + the two beings presented." + +This nightmare occupied some ten pages of manuscript and wound up with +a sermon so destructive of all hope to non-Presbyterians that it took +the first prize. This composition was considered to be the very finest +effort of the evening. The mayor of the village, in delivering the +prize to the author of it, made a warm speech in which he said that it +was by far the most "eloquent" thing he had ever listened to, and that +Daniel Webster himself might well be proud of it. + +It may be remarked, in passing, that the number of compositions in +which the word "beauteous" was over-fondled, and human experience +referred to as "life's page," was up to the usual average. + +Now the master, mellow almost to the verge of geniality, put his chair +aside, turned his back to the audience, and began to draw a map of +America on the blackboard, to exercise the geography class upon. But he +made a sad business of it with his unsteady hand, and a smothered +titter rippled over the house. He knew what the matter was, and set +himself to right it. He sponged out lines and remade them; but he only +distorted them more than ever, and the tittering was more pronounced. +He threw his entire attention upon his work, now, as if determined not +to be put down by the mirth. He felt that all eyes were fastened upon +him; he imagined he was succeeding, and yet the tittering continued; it +even manifestly increased. And well it might. There was a garret above, +pierced with a scuttle over his head; and down through this scuttle +came a cat, suspended around the haunches by a string; she had a rag +tied about her head and jaws to keep her from mewing; as she slowly +descended she curved upward and clawed at the string, she swung +downward and clawed at the intangible air. The tittering rose higher +and higher--the cat was within six inches of the absorbed teacher's +head--down, down, a little lower, and she grabbed his wig with her +desperate claws, clung to it, and was snatched up into the garret in an +instant with her trophy still in her possession! And how the light did +blaze abroad from the master's bald pate--for the sign-painter's boy +had GILDED it! + +That broke up the meeting. The boys were avenged. Vacation had come. + + NOTE:--The pretended "compositions" quoted in + this chapter are taken without alteration from a + volume entitled "Prose and Poetry, by a Western + Lady"--but they are exactly and precisely after + the schoolgirl pattern, and hence are much + happier than any mere imitations could be. + + + +CHAPTER XXII + +TOM joined the new order of Cadets of Temperance, being attracted by +the showy character of their "regalia." He promised to abstain from +smoking, chewing, and profanity as long as he remained a member. Now he +found out a new thing--namely, that to promise not to do a thing is the +surest way in the world to make a body want to go and do that very +thing. Tom soon found himself tormented with a desire to drink and +swear; the desire grew to be so intense that nothing but the hope of a +chance to display himself in his red sash kept him from withdrawing +from the order. Fourth of July was coming; but he soon gave that up +--gave it up before he had worn his shackles over forty-eight hours--and +fixed his hopes upon old Judge Frazer, justice of the peace, who was +apparently on his deathbed and would have a big public funeral, since +he was so high an official. During three days Tom was deeply concerned +about the Judge's condition and hungry for news of it. Sometimes his +hopes ran high--so high that he would venture to get out his regalia +and practise before the looking-glass. But the Judge had a most +discouraging way of fluctuating. At last he was pronounced upon the +mend--and then convalescent. Tom was disgusted; and felt a sense of +injury, too. He handed in his resignation at once--and that night the +Judge suffered a relapse and died. Tom resolved that he would never +trust a man like that again. + +The funeral was a fine thing. The Cadets paraded in a style calculated +to kill the late member with envy. Tom was a free boy again, however +--there was something in that. He could drink and swear, now--but found +to his surprise that he did not want to. The simple fact that he could, +took the desire away, and the charm of it. + +Tom presently wondered to find that his coveted vacation was beginning +to hang a little heavily on his hands. + +He attempted a diary--but nothing happened during three days, and so +he abandoned it. + +The first of all the negro minstrel shows came to town, and made a +sensation. Tom and Joe Harper got up a band of performers and were +happy for two days. + +Even the Glorious Fourth was in some sense a failure, for it rained +hard, there was no procession in consequence, and the greatest man in +the world (as Tom supposed), Mr. Benton, an actual United States +Senator, proved an overwhelming disappointment--for he was not +twenty-five feet high, nor even anywhere in the neighborhood of it. + +A circus came. The boys played circus for three days afterward in +tents made of rag carpeting--admission, three pins for boys, two for +girls--and then circusing was abandoned. + +A phrenologist and a mesmerizer came--and went again and left the +village duller and drearier than ever. + +There were some boys-and-girls' parties, but they were so few and so +delightful that they only made the aching voids between ache the harder. + +Becky Thatcher was gone to her Constantinople home to stay with her +parents during vacation--so there was no bright side to life anywhere. + +The dreadful secret of the murder was a chronic misery. It was a very +cancer for permanency and pain. + +Then came the measles. + +During two long weeks Tom lay a prisoner, dead to the world and its +happenings. He was very ill, he was interested in nothing. When he got +upon his feet at last and moved feebly down-town, a melancholy change +had come over everything and every creature. There had been a +"revival," and everybody had "got religion," not only the adults, but +even the boys and girls. Tom went about, hoping against hope for the +sight of one blessed sinful face, but disappointment crossed him +everywhere. He found Joe Harper studying a Testament, and turned sadly +away from the depressing spectacle. He sought Ben Rogers, and found him +visiting the poor with a basket of tracts. He hunted up Jim Hollis, who +called his attention to the precious blessing of his late measles as a +warning. Every boy he encountered added another ton to his depression; +and when, in desperation, he flew for refuge at last to the bosom of +Huckleberry Finn and was received with a Scriptural quotation, his +heart broke and he crept home and to bed realizing that he alone of all +the town was lost, forever and forever. + +And that night there came on a terrific storm, with driving rain, +awful claps of thunder and blinding sheets of lightning. He covered his +head with the bedclothes and waited in a horror of suspense for his +doom; for he had not the shadow of a doubt that all this hubbub was +about him. He believed he had taxed the forbearance of the powers above +to the extremity of endurance and that this was the result. It might +have seemed to him a waste of pomp and ammunition to kill a bug with a +battery of artillery, but there seemed nothing incongruous about the +getting up such an expensive thunderstorm as this to knock the turf +from under an insect like himself. + +By and by the tempest spent itself and died without accomplishing its +object. The boy's first impulse was to be grateful, and reform. His +second was to wait--for there might not be any more storms. + +The next day the doctors were back; Tom had relapsed. The three weeks +he spent on his back this time seemed an entire age. When he got abroad +at last he was hardly grateful that he had been spared, remembering how +lonely was his estate, how companionless and forlorn he was. He drifted +listlessly down the street and found Jim Hollis acting as judge in a +juvenile court that was trying a cat for murder, in the presence of her +victim, a bird. He found Joe Harper and Huck Finn up an alley eating a +stolen melon. Poor lads! they--like Tom--had suffered a relapse. + + + +CHAPTER XXIII + +AT last the sleepy atmosphere was stirred--and vigorously: the murder +trial came on in the court. It became the absorbing topic of village +talk immediately. Tom could not get away from it. Every reference to +the murder sent a shudder to his heart, for his troubled conscience and +fears almost persuaded him that these remarks were put forth in his +hearing as "feelers"; he did not see how he could be suspected of +knowing anything about the murder, but still he could not be +comfortable in the midst of this gossip. It kept him in a cold shiver +all the time. He took Huck to a lonely place to have a talk with him. +It would be some relief to unseal his tongue for a little while; to +divide his burden of distress with another sufferer. Moreover, he +wanted to assure himself that Huck had remained discreet. + +"Huck, have you ever told anybody about--that?" + +"'Bout what?" + +"You know what." + +"Oh--'course I haven't." + +"Never a word?" + +"Never a solitary word, so help me. What makes you ask?" + +"Well, I was afeard." + +"Why, Tom Sawyer, we wouldn't be alive two days if that got found out. +YOU know that." + +Tom felt more comfortable. After a pause: + +"Huck, they couldn't anybody get you to tell, could they?" + +"Get me to tell? Why, if I wanted that half-breed devil to drownd me +they could get me to tell. They ain't no different way." + +"Well, that's all right, then. I reckon we're safe as long as we keep +mum. But let's swear again, anyway. It's more surer." + +"I'm agreed." + +So they swore again with dread solemnities. + +"What is the talk around, Huck? I've heard a power of it." + +"Talk? Well, it's just Muff Potter, Muff Potter, Muff Potter all the +time. It keeps me in a sweat, constant, so's I want to hide som'ers." + +"That's just the same way they go on round me. I reckon he's a goner. +Don't you feel sorry for him, sometimes?" + +"Most always--most always. He ain't no account; but then he hain't +ever done anything to hurt anybody. Just fishes a little, to get money +to get drunk on--and loafs around considerable; but lord, we all do +that--leastways most of us--preachers and such like. But he's kind of +good--he give me half a fish, once, when there warn't enough for two; +and lots of times he's kind of stood by me when I was out of luck." + +"Well, he's mended kites for me, Huck, and knitted hooks on to my +line. I wish we could get him out of there." + +"My! we couldn't get him out, Tom. And besides, 'twouldn't do any +good; they'd ketch him again." + +"Yes--so they would. But I hate to hear 'em abuse him so like the +dickens when he never done--that." + +"I do too, Tom. Lord, I hear 'em say he's the bloodiest looking +villain in this country, and they wonder he wasn't ever hung before." + +"Yes, they talk like that, all the time. I've heard 'em say that if he +was to get free they'd lynch him." + +"And they'd do it, too." + +The boys had a long talk, but it brought them little comfort. As the +twilight drew on, they found themselves hanging about the neighborhood +of the little isolated jail, perhaps with an undefined hope that +something would happen that might clear away their difficulties. But +nothing happened; there seemed to be no angels or fairies interested in +this luckless captive. + +The boys did as they had often done before--went to the cell grating +and gave Potter some tobacco and matches. He was on the ground floor +and there were no guards. + +His gratitude for their gifts had always smote their consciences +before--it cut deeper than ever, this time. They felt cowardly and +treacherous to the last degree when Potter said: + +"You've been mighty good to me, boys--better'n anybody else in this +town. And I don't forget it, I don't. Often I says to myself, says I, +'I used to mend all the boys' kites and things, and show 'em where the +good fishin' places was, and befriend 'em what I could, and now they've +all forgot old Muff when he's in trouble; but Tom don't, and Huck +don't--THEY don't forget him, says I, 'and I don't forget them.' Well, +boys, I done an awful thing--drunk and crazy at the time--that's the +only way I account for it--and now I got to swing for it, and it's +right. Right, and BEST, too, I reckon--hope so, anyway. Well, we won't +talk about that. I don't want to make YOU feel bad; you've befriended +me. But what I want to say, is, don't YOU ever get drunk--then you won't +ever get here. Stand a litter furder west--so--that's it; it's a prime +comfort to see faces that's friendly when a body's in such a muck of +trouble, and there don't none come here but yourn. Good friendly +faces--good friendly faces. Git up on one another's backs and let me +touch 'em. That's it. Shake hands--yourn'll come through the bars, but +mine's too big. Little hands, and weak--but they've helped Muff Potter +a power, and they'd help him more if they could." + +Tom went home miserable, and his dreams that night were full of +horrors. The next day and the day after, he hung about the court-room, +drawn by an almost irresistible impulse to go in, but forcing himself +to stay out. Huck was having the same experience. They studiously +avoided each other. Each wandered away, from time to time, but the same +dismal fascination always brought them back presently. Tom kept his +ears open when idlers sauntered out of the court-room, but invariably +heard distressing news--the toils were closing more and more +relentlessly around poor Potter. At the end of the second day the +village talk was to the effect that Injun Joe's evidence stood firm and +unshaken, and that there was not the slightest question as to what the +jury's verdict would be. + +Tom was out late, that night, and came to bed through the window. He +was in a tremendous state of excitement. It was hours before he got to +sleep. All the village flocked to the court-house the next morning, for +this was to be the great day. Both sexes were about equally represented +in the packed audience. After a long wait the jury filed in and took +their places; shortly afterward, Potter, pale and haggard, timid and +hopeless, was brought in, with chains upon him, and seated where all +the curious eyes could stare at him; no less conspicuous was Injun Joe, +stolid as ever. There was another pause, and then the judge arrived and +the sheriff proclaimed the opening of the court. The usual whisperings +among the lawyers and gathering together of papers followed. These +details and accompanying delays worked up an atmosphere of preparation +that was as impressive as it was fascinating. + +Now a witness was called who testified that he found Muff Potter +washing in the brook, at an early hour of the morning that the murder +was discovered, and that he immediately sneaked away. After some +further questioning, counsel for the prosecution said: + +"Take the witness." + +The prisoner raised his eyes for a moment, but dropped them again when +his own counsel said: + +"I have no questions to ask him." + +The next witness proved the finding of the knife near the corpse. +Counsel for the prosecution said: + +"Take the witness." + +"I have no questions to ask him," Potter's lawyer replied. + +A third witness swore he had often seen the knife in Potter's +possession. + +"Take the witness." + +Counsel for Potter declined to question him. The faces of the audience +began to betray annoyance. Did this attorney mean to throw away his +client's life without an effort? + +Several witnesses deposed concerning Potter's guilty behavior when +brought to the scene of the murder. They were allowed to leave the +stand without being cross-questioned. + +Every detail of the damaging circumstances that occurred in the +graveyard upon that morning which all present remembered so well was +brought out by credible witnesses, but none of them were cross-examined +by Potter's lawyer. The perplexity and dissatisfaction of the house +expressed itself in murmurs and provoked a reproof from the bench. +Counsel for the prosecution now said: + +"By the oaths of citizens whose simple word is above suspicion, we +have fastened this awful crime, beyond all possibility of question, +upon the unhappy prisoner at the bar. We rest our case here." + +A groan escaped from poor Potter, and he put his face in his hands and +rocked his body softly to and fro, while a painful silence reigned in +the court-room. Many men were moved, and many women's compassion +testified itself in tears. Counsel for the defence rose and said: + +"Your honor, in our remarks at the opening of this trial, we +foreshadowed our purpose to prove that our client did this fearful deed +while under the influence of a blind and irresponsible delirium +produced by drink. We have changed our mind. We shall not offer that +plea." [Then to the clerk:] "Call Thomas Sawyer!" + +A puzzled amazement awoke in every face in the house, not even +excepting Potter's. Every eye fastened itself with wondering interest +upon Tom as he rose and took his place upon the stand. The boy looked +wild enough, for he was badly scared. The oath was administered. + +"Thomas Sawyer, where were you on the seventeenth of June, about the +hour of midnight?" + +Tom glanced at Injun Joe's iron face and his tongue failed him. The +audience listened breathless, but the words refused to come. After a +few moments, however, the boy got a little of his strength back, and +managed to put enough of it into his voice to make part of the house +hear: + +"In the graveyard!" + +"A little bit louder, please. Don't be afraid. You were--" + +"In the graveyard." + +A contemptuous smile flitted across Injun Joe's face. + +"Were you anywhere near Horse Williams' grave?" + +"Yes, sir." + +"Speak up--just a trifle louder. How near were you?" + +"Near as I am to you." + +"Were you hidden, or not?" + +"I was hid." + +"Where?" + +"Behind the elms that's on the edge of the grave." + +Injun Joe gave a barely perceptible start. + +"Any one with you?" + +"Yes, sir. I went there with--" + +"Wait--wait a moment. Never mind mentioning your companion's name. We +will produce him at the proper time. Did you carry anything there with +you." + +Tom hesitated and looked confused. + +"Speak out, my boy--don't be diffident. The truth is always +respectable. What did you take there?" + +"Only a--a--dead cat." + +There was a ripple of mirth, which the court checked. + +"We will produce the skeleton of that cat. Now, my boy, tell us +everything that occurred--tell it in your own way--don't skip anything, +and don't be afraid." + +Tom began--hesitatingly at first, but as he warmed to his subject his +words flowed more and more easily; in a little while every sound ceased +but his own voice; every eye fixed itself upon him; with parted lips +and bated breath the audience hung upon his words, taking no note of +time, rapt in the ghastly fascinations of the tale. The strain upon +pent emotion reached its climax when the boy said: + +"--and as the doctor fetched the board around and Muff Potter fell, +Injun Joe jumped with the knife and--" + +Crash! Quick as lightning the half-breed sprang for a window, tore his +way through all opposers, and was gone! + + + +CHAPTER XXIV + +TOM was a glittering hero once more--the pet of the old, the envy of +the young. His name even went into immortal print, for the village +paper magnified him. There were some that believed he would be +President, yet, if he escaped hanging. + +As usual, the fickle, unreasoning world took Muff Potter to its bosom +and fondled him as lavishly as it had abused him before. But that sort +of conduct is to the world's credit; therefore it is not well to find +fault with it. + +Tom's days were days of splendor and exultation to him, but his nights +were seasons of horror. Injun Joe infested all his dreams, and always +with doom in his eye. Hardly any temptation could persuade the boy to +stir abroad after nightfall. Poor Huck was in the same state of +wretchedness and terror, for Tom had told the whole story to the lawyer +the night before the great day of the trial, and Huck was sore afraid +that his share in the business might leak out, yet, notwithstanding +Injun Joe's flight had saved him the suffering of testifying in court. +The poor fellow had got the attorney to promise secrecy, but what of +that? Since Tom's harassed conscience had managed to drive him to the +lawyer's house by night and wring a dread tale from lips that had been +sealed with the dismalest and most formidable of oaths, Huck's +confidence in the human race was well-nigh obliterated. + +Daily Muff Potter's gratitude made Tom glad he had spoken; but nightly +he wished he had sealed up his tongue. + +Half the time Tom was afraid Injun Joe would never be captured; the +other half he was afraid he would be. He felt sure he never could draw +a safe breath again until that man was dead and he had seen the corpse. + +Rewards had been offered, the country had been scoured, but no Injun +Joe was found. One of those omniscient and awe-inspiring marvels, a +detective, came up from St. Louis, moused around, shook his head, +looked wise, and made that sort of astounding success which members of +that craft usually achieve. That is to say, he "found a clew." But you +can't hang a "clew" for murder, and so after that detective had got +through and gone home, Tom felt just as insecure as he was before. + +The slow days drifted on, and each left behind it a slightly lightened +weight of apprehension. + + + +CHAPTER XXV + +THERE comes a time in every rightly-constructed boy's life when he has +a raging desire to go somewhere and dig for hidden treasure. This +desire suddenly came upon Tom one day. He sallied out to find Joe +Harper, but failed of success. Next he sought Ben Rogers; he had gone +fishing. Presently he stumbled upon Huck Finn the Red-Handed. Huck +would answer. Tom took him to a private place and opened the matter to +him confidentially. Huck was willing. Huck was always willing to take a +hand in any enterprise that offered entertainment and required no +capital, for he had a troublesome superabundance of that sort of time +which is not money. "Where'll we dig?" said Huck. + +"Oh, most anywhere." + +"Why, is it hid all around?" + +"No, indeed it ain't. It's hid in mighty particular places, Huck +--sometimes on islands, sometimes in rotten chests under the end of a +limb of an old dead tree, just where the shadow falls at midnight; but +mostly under the floor in ha'nted houses." + +"Who hides it?" + +"Why, robbers, of course--who'd you reckon? Sunday-school +sup'rintendents?" + +"I don't know. If 'twas mine I wouldn't hide it; I'd spend it and have +a good time." + +"So would I. But robbers don't do that way. They always hide it and +leave it there." + +"Don't they come after it any more?" + +"No, they think they will, but they generally forget the marks, or +else they die. Anyway, it lays there a long time and gets rusty; and by +and by somebody finds an old yellow paper that tells how to find the +marks--a paper that's got to be ciphered over about a week because it's +mostly signs and hy'roglyphics." + +"Hyro--which?" + +"Hy'roglyphics--pictures and things, you know, that don't seem to mean +anything." + +"Have you got one of them papers, Tom?" + +"No." + +"Well then, how you going to find the marks?" + +"I don't want any marks. They always bury it under a ha'nted house or +on an island, or under a dead tree that's got one limb sticking out. +Well, we've tried Jackson's Island a little, and we can try it again +some time; and there's the old ha'nted house up the Still-House branch, +and there's lots of dead-limb trees--dead loads of 'em." + +"Is it under all of them?" + +"How you talk! No!" + +"Then how you going to know which one to go for?" + +"Go for all of 'em!" + +"Why, Tom, it'll take all summer." + +"Well, what of that? Suppose you find a brass pot with a hundred +dollars in it, all rusty and gray, or rotten chest full of di'monds. +How's that?" + +Huck's eyes glowed. + +"That's bully. Plenty bully enough for me. Just you gimme the hundred +dollars and I don't want no di'monds." + +"All right. But I bet you I ain't going to throw off on di'monds. Some +of 'em's worth twenty dollars apiece--there ain't any, hardly, but's +worth six bits or a dollar." + +"No! Is that so?" + +"Cert'nly--anybody'll tell you so. Hain't you ever seen one, Huck?" + +"Not as I remember." + +"Oh, kings have slathers of them." + +"Well, I don' know no kings, Tom." + +"I reckon you don't. But if you was to go to Europe you'd see a raft +of 'em hopping around." + +"Do they hop?" + +"Hop?--your granny! No!" + +"Well, what did you say they did, for?" + +"Shucks, I only meant you'd SEE 'em--not hopping, of course--what do +they want to hop for?--but I mean you'd just see 'em--scattered around, +you know, in a kind of a general way. Like that old humpbacked Richard." + +"Richard? What's his other name?" + +"He didn't have any other name. Kings don't have any but a given name." + +"No?" + +"But they don't." + +"Well, if they like it, Tom, all right; but I don't want to be a king +and have only just a given name, like a nigger. But say--where you +going to dig first?" + +"Well, I don't know. S'pose we tackle that old dead-limb tree on the +hill t'other side of Still-House branch?" + +"I'm agreed." + +So they got a crippled pick and a shovel, and set out on their +three-mile tramp. They arrived hot and panting, and threw themselves +down in the shade of a neighboring elm to rest and have a smoke. + +"I like this," said Tom. + +"So do I." + +"Say, Huck, if we find a treasure here, what you going to do with your +share?" + +"Well, I'll have pie and a glass of soda every day, and I'll go to +every circus that comes along. I bet I'll have a gay time." + +"Well, ain't you going to save any of it?" + +"Save it? What for?" + +"Why, so as to have something to live on, by and by." + +"Oh, that ain't any use. Pap would come back to thish-yer town some +day and get his claws on it if I didn't hurry up, and I tell you he'd +clean it out pretty quick. What you going to do with yourn, Tom?" + +"I'm going to buy a new drum, and a sure-'nough sword, and a red +necktie and a bull pup, and get married." + +"Married!" + +"That's it." + +"Tom, you--why, you ain't in your right mind." + +"Wait--you'll see." + +"Well, that's the foolishest thing you could do. Look at pap and my +mother. Fight! Why, they used to fight all the time. I remember, mighty +well." + +"That ain't anything. The girl I'm going to marry won't fight." + +"Tom, I reckon they're all alike. They'll all comb a body. Now you +better think 'bout this awhile. I tell you you better. What's the name +of the gal?" + +"It ain't a gal at all--it's a girl." + +"It's all the same, I reckon; some says gal, some says girl--both's +right, like enough. Anyway, what's her name, Tom?" + +"I'll tell you some time--not now." + +"All right--that'll do. Only if you get married I'll be more lonesomer +than ever." + +"No you won't. You'll come and live with me. Now stir out of this and +we'll go to digging." + +They worked and sweated for half an hour. No result. They toiled +another half-hour. Still no result. Huck said: + +"Do they always bury it as deep as this?" + +"Sometimes--not always. Not generally. I reckon we haven't got the +right place." + +So they chose a new spot and began again. The labor dragged a little, +but still they made progress. They pegged away in silence for some +time. Finally Huck leaned on his shovel, swabbed the beaded drops from +his brow with his sleeve, and said: + +"Where you going to dig next, after we get this one?" + +"I reckon maybe we'll tackle the old tree that's over yonder on +Cardiff Hill back of the widow's." + +"I reckon that'll be a good one. But won't the widow take it away from +us, Tom? It's on her land." + +"SHE take it away! Maybe she'd like to try it once. Whoever finds one +of these hid treasures, it belongs to him. It don't make any difference +whose land it's on." + +That was satisfactory. The work went on. By and by Huck said: + +"Blame it, we must be in the wrong place again. What do you think?" + +"It is mighty curious, Huck. I don't understand it. Sometimes witches +interfere. I reckon maybe that's what's the trouble now." + +"Shucks! Witches ain't got no power in the daytime." + +"Well, that's so. I didn't think of that. Oh, I know what the matter +is! What a blamed lot of fools we are! You got to find out where the +shadow of the limb falls at midnight, and that's where you dig!" + +"Then consound it, we've fooled away all this work for nothing. Now +hang it all, we got to come back in the night. It's an awful long way. +Can you get out?" + +"I bet I will. We've got to do it to-night, too, because if somebody +sees these holes they'll know in a minute what's here and they'll go +for it." + +"Well, I'll come around and maow to-night." + +"All right. Let's hide the tools in the bushes." + +The boys were there that night, about the appointed time. They sat in +the shadow waiting. It was a lonely place, and an hour made solemn by +old traditions. Spirits whispered in the rustling leaves, ghosts lurked +in the murky nooks, the deep baying of a hound floated up out of the +distance, an owl answered with his sepulchral note. The boys were +subdued by these solemnities, and talked little. By and by they judged +that twelve had come; they marked where the shadow fell, and began to +dig. Their hopes commenced to rise. Their interest grew stronger, and +their industry kept pace with it. The hole deepened and still deepened, +but every time their hearts jumped to hear the pick strike upon +something, they only suffered a new disappointment. It was only a stone +or a chunk. At last Tom said: + +"It ain't any use, Huck, we're wrong again." + +"Well, but we CAN'T be wrong. We spotted the shadder to a dot." + +"I know it, but then there's another thing." + +"What's that?". + +"Why, we only guessed at the time. Like enough it was too late or too +early." + +Huck dropped his shovel. + +"That's it," said he. "That's the very trouble. We got to give this +one up. We can't ever tell the right time, and besides this kind of +thing's too awful, here this time of night with witches and ghosts +a-fluttering around so. I feel as if something's behind me all the time; +and I'm afeard to turn around, becuz maybe there's others in front +a-waiting for a chance. I been creeping all over, ever since I got here." + +"Well, I've been pretty much so, too, Huck. They most always put in a +dead man when they bury a treasure under a tree, to look out for it." + +"Lordy!" + +"Yes, they do. I've always heard that." + +"Tom, I don't like to fool around much where there's dead people. A +body's bound to get into trouble with 'em, sure." + +"I don't like to stir 'em up, either. S'pose this one here was to +stick his skull out and say something!" + +"Don't Tom! It's awful." + +"Well, it just is. Huck, I don't feel comfortable a bit." + +"Say, Tom, let's give this place up, and try somewheres else." + +"All right, I reckon we better." + +"What'll it be?" + +Tom considered awhile; and then said: + +"The ha'nted house. That's it!" + +"Blame it, I don't like ha'nted houses, Tom. Why, they're a dern sight +worse'n dead people. Dead people might talk, maybe, but they don't come +sliding around in a shroud, when you ain't noticing, and peep over your +shoulder all of a sudden and grit their teeth, the way a ghost does. I +couldn't stand such a thing as that, Tom--nobody could." + +"Yes, but, Huck, ghosts don't travel around only at night. They won't +hender us from digging there in the daytime." + +"Well, that's so. But you know mighty well people don't go about that +ha'nted house in the day nor the night." + +"Well, that's mostly because they don't like to go where a man's been +murdered, anyway--but nothing's ever been seen around that house except +in the night--just some blue lights slipping by the windows--no regular +ghosts." + +"Well, where you see one of them blue lights flickering around, Tom, +you can bet there's a ghost mighty close behind it. It stands to +reason. Becuz you know that they don't anybody but ghosts use 'em." + +"Yes, that's so. But anyway they don't come around in the daytime, so +what's the use of our being afeard?" + +"Well, all right. We'll tackle the ha'nted house if you say so--but I +reckon it's taking chances." + +They had started down the hill by this time. There in the middle of +the moonlit valley below them stood the "ha'nted" house, utterly +isolated, its fences gone long ago, rank weeds smothering the very +doorsteps, the chimney crumbled to ruin, the window-sashes vacant, a +corner of the roof caved in. The boys gazed awhile, half expecting to +see a blue light flit past a window; then talking in a low tone, as +befitted the time and the circumstances, they struck far off to the +right, to give the haunted house a wide berth, and took their way +homeward through the woods that adorned the rearward side of Cardiff +Hill. + + + +CHAPTER XXVI + +ABOUT noon the next day the boys arrived at the dead tree; they had +come for their tools. Tom was impatient to go to the haunted house; +Huck was measurably so, also--but suddenly said: + +"Lookyhere, Tom, do you know what day it is?" + +Tom mentally ran over the days of the week, and then quickly lifted +his eyes with a startled look in them-- + +"My! I never once thought of it, Huck!" + +"Well, I didn't neither, but all at once it popped onto me that it was +Friday." + +"Blame it, a body can't be too careful, Huck. We might 'a' got into an +awful scrape, tackling such a thing on a Friday." + +"MIGHT! Better say we WOULD! There's some lucky days, maybe, but +Friday ain't." + +"Any fool knows that. I don't reckon YOU was the first that found it +out, Huck." + +"Well, I never said I was, did I? And Friday ain't all, neither. I had +a rotten bad dream last night--dreampt about rats." + +"No! Sure sign of trouble. Did they fight?" + +"No." + +"Well, that's good, Huck. When they don't fight it's only a sign that +there's trouble around, you know. All we got to do is to look mighty +sharp and keep out of it. We'll drop this thing for to-day, and play. +Do you know Robin Hood, Huck?" + +"No. Who's Robin Hood?" + +"Why, he was one of the greatest men that was ever in England--and the +best. He was a robber." + +"Cracky, I wisht I was. Who did he rob?" + +"Only sheriffs and bishops and rich people and kings, and such like. +But he never bothered the poor. He loved 'em. He always divided up with +'em perfectly square." + +"Well, he must 'a' been a brick." + +"I bet you he was, Huck. Oh, he was the noblest man that ever was. +They ain't any such men now, I can tell you. He could lick any man in +England, with one hand tied behind him; and he could take his yew bow +and plug a ten-cent piece every time, a mile and a half." + +"What's a YEW bow?" + +"I don't know. It's some kind of a bow, of course. And if he hit that +dime only on the edge he would set down and cry--and curse. But we'll +play Robin Hood--it's nobby fun. I'll learn you." + +"I'm agreed." + +So they played Robin Hood all the afternoon, now and then casting a +yearning eye down upon the haunted house and passing a remark about the +morrow's prospects and possibilities there. As the sun began to sink +into the west they took their way homeward athwart the long shadows of +the trees and soon were buried from sight in the forests of Cardiff +Hill. + +On Saturday, shortly after noon, the boys were at the dead tree again. +They had a smoke and a chat in the shade, and then dug a little in +their last hole, not with great hope, but merely because Tom said there +were so many cases where people had given up a treasure after getting +down within six inches of it, and then somebody else had come along and +turned it up with a single thrust of a shovel. The thing failed this +time, however, so the boys shouldered their tools and went away feeling +that they had not trifled with fortune, but had fulfilled all the +requirements that belong to the business of treasure-hunting. + +When they reached the haunted house there was something so weird and +grisly about the dead silence that reigned there under the baking sun, +and something so depressing about the loneliness and desolation of the +place, that they were afraid, for a moment, to venture in. Then they +crept to the door and took a trembling peep. They saw a weed-grown, +floorless room, unplastered, an ancient fireplace, vacant windows, a +ruinous staircase; and here, there, and everywhere hung ragged and +abandoned cobwebs. They presently entered, softly, with quickened +pulses, talking in whispers, ears alert to catch the slightest sound, +and muscles tense and ready for instant retreat. + +In a little while familiarity modified their fears and they gave the +place a critical and interested examination, rather admiring their own +boldness, and wondering at it, too. Next they wanted to look up-stairs. +This was something like cutting off retreat, but they got to daring +each other, and of course there could be but one result--they threw +their tools into a corner and made the ascent. Up there were the same +signs of decay. In one corner they found a closet that promised +mystery, but the promise was a fraud--there was nothing in it. Their +courage was up now and well in hand. They were about to go down and +begin work when-- + +"Sh!" said Tom. + +"What is it?" whispered Huck, blanching with fright. + +"Sh!... There!... Hear it?" + +"Yes!... Oh, my! Let's run!" + +"Keep still! Don't you budge! They're coming right toward the door." + +The boys stretched themselves upon the floor with their eyes to +knot-holes in the planking, and lay waiting, in a misery of fear. + +"They've stopped.... No--coming.... Here they are. Don't whisper +another word, Huck. My goodness, I wish I was out of this!" + +Two men entered. Each boy said to himself: "There's the old deaf and +dumb Spaniard that's been about town once or twice lately--never saw +t'other man before." + +"T'other" was a ragged, unkempt creature, with nothing very pleasant +in his face. The Spaniard was wrapped in a serape; he had bushy white +whiskers; long white hair flowed from under his sombrero, and he wore +green goggles. When they came in, "t'other" was talking in a low voice; +they sat down on the ground, facing the door, with their backs to the +wall, and the speaker continued his remarks. His manner became less +guarded and his words more distinct as he proceeded: + +"No," said he, "I've thought it all over, and I don't like it. It's +dangerous." + +"Dangerous!" grunted the "deaf and dumb" Spaniard--to the vast +surprise of the boys. "Milksop!" + +This voice made the boys gasp and quake. It was Injun Joe's! There was +silence for some time. Then Joe said: + +"What's any more dangerous than that job up yonder--but nothing's come +of it." + +"That's different. Away up the river so, and not another house about. +'Twon't ever be known that we tried, anyway, long as we didn't succeed." + +"Well, what's more dangerous than coming here in the daytime!--anybody +would suspicion us that saw us." + +"I know that. But there warn't any other place as handy after that +fool of a job. I want to quit this shanty. I wanted to yesterday, only +it warn't any use trying to stir out of here, with those infernal boys +playing over there on the hill right in full view." + +"Those infernal boys" quaked again under the inspiration of this +remark, and thought how lucky it was that they had remembered it was +Friday and concluded to wait a day. They wished in their hearts they +had waited a year. + +The two men got out some food and made a luncheon. After a long and +thoughtful silence, Injun Joe said: + +"Look here, lad--you go back up the river where you belong. Wait there +till you hear from me. I'll take the chances on dropping into this town +just once more, for a look. We'll do that 'dangerous' job after I've +spied around a little and think things look well for it. Then for +Texas! We'll leg it together!" + +This was satisfactory. Both men presently fell to yawning, and Injun +Joe said: + +"I'm dead for sleep! It's your turn to watch." + +He curled down in the weeds and soon began to snore. His comrade +stirred him once or twice and he became quiet. Presently the watcher +began to nod; his head drooped lower and lower, both men began to snore +now. + +The boys drew a long, grateful breath. Tom whispered: + +"Now's our chance--come!" + +Huck said: + +"I can't--I'd die if they was to wake." + +Tom urged--Huck held back. At last Tom rose slowly and softly, and +started alone. But the first step he made wrung such a hideous creak +from the crazy floor that he sank down almost dead with fright. He +never made a second attempt. The boys lay there counting the dragging +moments till it seemed to them that time must be done and eternity +growing gray; and then they were grateful to note that at last the sun +was setting. + +Now one snore ceased. Injun Joe sat up, stared around--smiled grimly +upon his comrade, whose head was drooping upon his knees--stirred him +up with his foot and said: + +"Here! YOU'RE a watchman, ain't you! All right, though--nothing's +happened." + +"My! have I been asleep?" + +"Oh, partly, partly. Nearly time for us to be moving, pard. What'll we +do with what little swag we've got left?" + +"I don't know--leave it here as we've always done, I reckon. No use to +take it away till we start south. Six hundred and fifty in silver's +something to carry." + +"Well--all right--it won't matter to come here once more." + +"No--but I'd say come in the night as we used to do--it's better." + +"Yes: but look here; it may be a good while before I get the right +chance at that job; accidents might happen; 'tain't in such a very good +place; we'll just regularly bury it--and bury it deep." + +"Good idea," said the comrade, who walked across the room, knelt down, +raised one of the rearward hearth-stones and took out a bag that +jingled pleasantly. He subtracted from it twenty or thirty dollars for +himself and as much for Injun Joe, and passed the bag to the latter, +who was on his knees in the corner, now, digging with his bowie-knife. + +The boys forgot all their fears, all their miseries in an instant. +With gloating eyes they watched every movement. Luck!--the splendor of +it was beyond all imagination! Six hundred dollars was money enough to +make half a dozen boys rich! Here was treasure-hunting under the +happiest auspices--there would not be any bothersome uncertainty as to +where to dig. They nudged each other every moment--eloquent nudges and +easily understood, for they simply meant--"Oh, but ain't you glad NOW +we're here!" + +Joe's knife struck upon something. + +"Hello!" said he. + +"What is it?" said his comrade. + +"Half-rotten plank--no, it's a box, I believe. Here--bear a hand and +we'll see what it's here for. Never mind, I've broke a hole." + +He reached his hand in and drew it out-- + +"Man, it's money!" + +The two men examined the handful of coins. They were gold. The boys +above were as excited as themselves, and as delighted. + +Joe's comrade said: + +"We'll make quick work of this. There's an old rusty pick over amongst +the weeds in the corner the other side of the fireplace--I saw it a +minute ago." + +He ran and brought the boys' pick and shovel. Injun Joe took the pick, +looked it over critically, shook his head, muttered something to +himself, and then began to use it. The box was soon unearthed. It was +not very large; it was iron bound and had been very strong before the +slow years had injured it. The men contemplated the treasure awhile in +blissful silence. + +"Pard, there's thousands of dollars here," said Injun Joe. + +"'Twas always said that Murrel's gang used to be around here one +summer," the stranger observed. + +"I know it," said Injun Joe; "and this looks like it, I should say." + +"Now you won't need to do that job." + +The half-breed frowned. Said he: + +"You don't know me. Least you don't know all about that thing. 'Tain't +robbery altogether--it's REVENGE!" and a wicked light flamed in his +eyes. "I'll need your help in it. When it's finished--then Texas. Go +home to your Nance and your kids, and stand by till you hear from me." + +"Well--if you say so; what'll we do with this--bury it again?" + +"Yes. [Ravishing delight overhead.] NO! by the great Sachem, no! +[Profound distress overhead.] I'd nearly forgot. That pick had fresh +earth on it! [The boys were sick with terror in a moment.] What +business has a pick and a shovel here? What business with fresh earth +on them? Who brought them here--and where are they gone? Have you heard +anybody?--seen anybody? What! bury it again and leave them to come and +see the ground disturbed? Not exactly--not exactly. We'll take it to my +den." + +"Why, of course! Might have thought of that before. You mean Number +One?" + +"No--Number Two--under the cross. The other place is bad--too common." + +"All right. It's nearly dark enough to start." + +Injun Joe got up and went about from window to window cautiously +peeping out. Presently he said: + +"Who could have brought those tools here? Do you reckon they can be +up-stairs?" + +The boys' breath forsook them. Injun Joe put his hand on his knife, +halted a moment, undecided, and then turned toward the stairway. The +boys thought of the closet, but their strength was gone. The steps came +creaking up the stairs--the intolerable distress of the situation woke +the stricken resolution of the lads--they were about to spring for the +closet, when there was a crash of rotten timbers and Injun Joe landed +on the ground amid the debris of the ruined stairway. He gathered +himself up cursing, and his comrade said: + +"Now what's the use of all that? If it's anybody, and they're up +there, let them STAY there--who cares? If they want to jump down, now, +and get into trouble, who objects? It will be dark in fifteen minutes +--and then let them follow us if they want to. I'm willing. In my +opinion, whoever hove those things in here caught a sight of us and +took us for ghosts or devils or something. I'll bet they're running +yet." + +Joe grumbled awhile; then he agreed with his friend that what daylight +was left ought to be economized in getting things ready for leaving. +Shortly afterward they slipped out of the house in the deepening +twilight, and moved toward the river with their precious box. + +Tom and Huck rose up, weak but vastly relieved, and stared after them +through the chinks between the logs of the house. Follow? Not they. +They were content to reach ground again without broken necks, and take +the townward track over the hill. They did not talk much. They were too +much absorbed in hating themselves--hating the ill luck that made them +take the spade and the pick there. But for that, Injun Joe never would +have suspected. He would have hidden the silver with the gold to wait +there till his "revenge" was satisfied, and then he would have had the +misfortune to find that money turn up missing. Bitter, bitter luck that +the tools were ever brought there! + +They resolved to keep a lookout for that Spaniard when he should come +to town spying out for chances to do his revengeful job, and follow him +to "Number Two," wherever that might be. Then a ghastly thought +occurred to Tom. + +"Revenge? What if he means US, Huck!" + +"Oh, don't!" said Huck, nearly fainting. + +They talked it all over, and as they entered town they agreed to +believe that he might possibly mean somebody else--at least that he +might at least mean nobody but Tom, since only Tom had testified. + +Very, very small comfort it was to Tom to be alone in danger! Company +would be a palpable improvement, he thought. + + + +CHAPTER XXVII + +THE adventure of the day mightily tormented Tom's dreams that night. +Four times he had his hands on that rich treasure and four times it +wasted to nothingness in his fingers as sleep forsook him and +wakefulness brought back the hard reality of his misfortune. As he lay +in the early morning recalling the incidents of his great adventure, he +noticed that they seemed curiously subdued and far away--somewhat as if +they had happened in another world, or in a time long gone by. Then it +occurred to him that the great adventure itself must be a dream! There +was one very strong argument in favor of this idea--namely, that the +quantity of coin he had seen was too vast to be real. He had never seen +as much as fifty dollars in one mass before, and he was like all boys +of his age and station in life, in that he imagined that all references +to "hundreds" and "thousands" were mere fanciful forms of speech, and +that no such sums really existed in the world. He never had supposed +for a moment that so large a sum as a hundred dollars was to be found +in actual money in any one's possession. If his notions of hidden +treasure had been analyzed, they would have been found to consist of a +handful of real dimes and a bushel of vague, splendid, ungraspable +dollars. + +But the incidents of his adventure grew sensibly sharper and clearer +under the attrition of thinking them over, and so he presently found +himself leaning to the impression that the thing might not have been a +dream, after all. This uncertainty must be swept away. He would snatch +a hurried breakfast and go and find Huck. Huck was sitting on the +gunwale of a flatboat, listlessly dangling his feet in the water and +looking very melancholy. Tom concluded to let Huck lead up to the +subject. If he did not do it, then the adventure would be proved to +have been only a dream. + +"Hello, Huck!" + +"Hello, yourself." + +Silence, for a minute. + +"Tom, if we'd 'a' left the blame tools at the dead tree, we'd 'a' got +the money. Oh, ain't it awful!" + +"'Tain't a dream, then, 'tain't a dream! Somehow I most wish it was. +Dog'd if I don't, Huck." + +"What ain't a dream?" + +"Oh, that thing yesterday. I been half thinking it was." + +"Dream! If them stairs hadn't broke down you'd 'a' seen how much dream +it was! I've had dreams enough all night--with that patch-eyed Spanish +devil going for me all through 'em--rot him!" + +"No, not rot him. FIND him! Track the money!" + +"Tom, we'll never find him. A feller don't have only one chance for +such a pile--and that one's lost. I'd feel mighty shaky if I was to see +him, anyway." + +"Well, so'd I; but I'd like to see him, anyway--and track him out--to +his Number Two." + +"Number Two--yes, that's it. I been thinking 'bout that. But I can't +make nothing out of it. What do you reckon it is?" + +"I dono. It's too deep. Say, Huck--maybe it's the number of a house!" + +"Goody!... No, Tom, that ain't it. If it is, it ain't in this +one-horse town. They ain't no numbers here." + +"Well, that's so. Lemme think a minute. Here--it's the number of a +room--in a tavern, you know!" + +"Oh, that's the trick! They ain't only two taverns. We can find out +quick." + +"You stay here, Huck, till I come." + +Tom was off at once. He did not care to have Huck's company in public +places. He was gone half an hour. He found that in the best tavern, No. +2 had long been occupied by a young lawyer, and was still so occupied. +In the less ostentatious house, No. 2 was a mystery. The +tavern-keeper's young son said it was kept locked all the time, and he +never saw anybody go into it or come out of it except at night; he did +not know any particular reason for this state of things; had had some +little curiosity, but it was rather feeble; had made the most of the +mystery by entertaining himself with the idea that that room was +"ha'nted"; had noticed that there was a light in there the night before. + +"That's what I've found out, Huck. I reckon that's the very No. 2 +we're after." + +"I reckon it is, Tom. Now what you going to do?" + +"Lemme think." + +Tom thought a long time. Then he said: + +"I'll tell you. The back door of that No. 2 is the door that comes out +into that little close alley between the tavern and the old rattle trap +of a brick store. Now you get hold of all the door-keys you can find, +and I'll nip all of auntie's, and the first dark night we'll go there +and try 'em. And mind you, keep a lookout for Injun Joe, because he +said he was going to drop into town and spy around once more for a +chance to get his revenge. If you see him, you just follow him; and if +he don't go to that No. 2, that ain't the place." + +"Lordy, I don't want to foller him by myself!" + +"Why, it'll be night, sure. He mightn't ever see you--and if he did, +maybe he'd never think anything." + +"Well, if it's pretty dark I reckon I'll track him. I dono--I dono. +I'll try." + +"You bet I'll follow him, if it's dark, Huck. Why, he might 'a' found +out he couldn't get his revenge, and be going right after that money." + +"It's so, Tom, it's so. I'll foller him; I will, by jingoes!" + +"Now you're TALKING! Don't you ever weaken, Huck, and I won't." + + + +CHAPTER XXVIII + +THAT night Tom and Huck were ready for their adventure. They hung +about the neighborhood of the tavern until after nine, one watching the +alley at a distance and the other the tavern door. Nobody entered the +alley or left it; nobody resembling the Spaniard entered or left the +tavern door. The night promised to be a fair one; so Tom went home with +the understanding that if a considerable degree of darkness came on, +Huck was to come and "maow," whereupon he would slip out and try the +keys. But the night remained clear, and Huck closed his watch and +retired to bed in an empty sugar hogshead about twelve. + +Tuesday the boys had the same ill luck. Also Wednesday. But Thursday +night promised better. Tom slipped out in good season with his aunt's +old tin lantern, and a large towel to blindfold it with. He hid the +lantern in Huck's sugar hogshead and the watch began. An hour before +midnight the tavern closed up and its lights (the only ones +thereabouts) were put out. No Spaniard had been seen. Nobody had +entered or left the alley. Everything was auspicious. The blackness of +darkness reigned, the perfect stillness was interrupted only by +occasional mutterings of distant thunder. + +Tom got his lantern, lit it in the hogshead, wrapped it closely in the +towel, and the two adventurers crept in the gloom toward the tavern. +Huck stood sentry and Tom felt his way into the alley. Then there was a +season of waiting anxiety that weighed upon Huck's spirits like a +mountain. He began to wish he could see a flash from the lantern--it +would frighten him, but it would at least tell him that Tom was alive +yet. It seemed hours since Tom had disappeared. Surely he must have +fainted; maybe he was dead; maybe his heart had burst under terror and +excitement. In his uneasiness Huck found himself drawing closer and +closer to the alley; fearing all sorts of dreadful things, and +momentarily expecting some catastrophe to happen that would take away +his breath. There was not much to take away, for he seemed only able to +inhale it by thimblefuls, and his heart would soon wear itself out, the +way it was beating. Suddenly there was a flash of light and Tom came +tearing by him: "Run!" said he; "run, for your life!" + +He needn't have repeated it; once was enough; Huck was making thirty +or forty miles an hour before the repetition was uttered. The boys +never stopped till they reached the shed of a deserted slaughter-house +at the lower end of the village. Just as they got within its shelter +the storm burst and the rain poured down. As soon as Tom got his breath +he said: + +"Huck, it was awful! I tried two of the keys, just as soft as I could; +but they seemed to make such a power of racket that I couldn't hardly +get my breath I was so scared. They wouldn't turn in the lock, either. +Well, without noticing what I was doing, I took hold of the knob, and +open comes the door! It warn't locked! I hopped in, and shook off the +towel, and, GREAT CAESAR'S GHOST!" + +"What!--what'd you see, Tom?" + +"Huck, I most stepped onto Injun Joe's hand!" + +"No!" + +"Yes! He was lying there, sound asleep on the floor, with his old +patch on his eye and his arms spread out." + +"Lordy, what did you do? Did he wake up?" + +"No, never budged. Drunk, I reckon. I just grabbed that towel and +started!" + +"I'd never 'a' thought of the towel, I bet!" + +"Well, I would. My aunt would make me mighty sick if I lost it." + +"Say, Tom, did you see that box?" + +"Huck, I didn't wait to look around. I didn't see the box, I didn't +see the cross. I didn't see anything but a bottle and a tin cup on the +floor by Injun Joe; yes, I saw two barrels and lots more bottles in the +room. Don't you see, now, what's the matter with that ha'nted room?" + +"How?" + +"Why, it's ha'nted with whiskey! Maybe ALL the Temperance Taverns have +got a ha'nted room, hey, Huck?" + +"Well, I reckon maybe that's so. Who'd 'a' thought such a thing? But +say, Tom, now's a mighty good time to get that box, if Injun Joe's +drunk." + +"It is, that! You try it!" + +Huck shuddered. + +"Well, no--I reckon not." + +"And I reckon not, Huck. Only one bottle alongside of Injun Joe ain't +enough. If there'd been three, he'd be drunk enough and I'd do it." + +There was a long pause for reflection, and then Tom said: + +"Lookyhere, Huck, less not try that thing any more till we know Injun +Joe's not in there. It's too scary. Now, if we watch every night, we'll +be dead sure to see him go out, some time or other, and then we'll +snatch that box quicker'n lightning." + +"Well, I'm agreed. I'll watch the whole night long, and I'll do it +every night, too, if you'll do the other part of the job." + +"All right, I will. All you got to do is to trot up Hooper Street a +block and maow--and if I'm asleep, you throw some gravel at the window +and that'll fetch me." + +"Agreed, and good as wheat!" + +"Now, Huck, the storm's over, and I'll go home. It'll begin to be +daylight in a couple of hours. You go back and watch that long, will +you?" + +"I said I would, Tom, and I will. I'll ha'nt that tavern every night +for a year! I'll sleep all day and I'll stand watch all night." + +"That's all right. Now, where you going to sleep?" + +"In Ben Rogers' hayloft. He lets me, and so does his pap's nigger man, +Uncle Jake. I tote water for Uncle Jake whenever he wants me to, and +any time I ask him he gives me a little something to eat if he can +spare it. That's a mighty good nigger, Tom. He likes me, becuz I don't +ever act as if I was above him. Sometime I've set right down and eat +WITH him. But you needn't tell that. A body's got to do things when +he's awful hungry he wouldn't want to do as a steady thing." + +"Well, if I don't want you in the daytime, I'll let you sleep. I won't +come bothering around. Any time you see something's up, in the night, +just skip right around and maow." + + + +CHAPTER XXIX + +THE first thing Tom heard on Friday morning was a glad piece of news +--Judge Thatcher's family had come back to town the night before. Both +Injun Joe and the treasure sunk into secondary importance for a moment, +and Becky took the chief place in the boy's interest. He saw her and +they had an exhausting good time playing "hi-spy" and "gully-keeper" +with a crowd of their school-mates. The day was completed and crowned +in a peculiarly satisfactory way: Becky teased her mother to appoint +the next day for the long-promised and long-delayed picnic, and she +consented. The child's delight was boundless; and Tom's not more +moderate. The invitations were sent out before sunset, and straightway +the young folks of the village were thrown into a fever of preparation +and pleasurable anticipation. Tom's excitement enabled him to keep +awake until a pretty late hour, and he had good hopes of hearing Huck's +"maow," and of having his treasure to astonish Becky and the picnickers +with, next day; but he was disappointed. No signal came that night. + +Morning came, eventually, and by ten or eleven o'clock a giddy and +rollicking company were gathered at Judge Thatcher's, and everything +was ready for a start. It was not the custom for elderly people to mar +the picnics with their presence. The children were considered safe +enough under the wings of a few young ladies of eighteen and a few +young gentlemen of twenty-three or thereabouts. The old steam ferryboat +was chartered for the occasion; presently the gay throng filed up the +main street laden with provision-baskets. Sid was sick and had to miss +the fun; Mary remained at home to entertain him. The last thing Mrs. +Thatcher said to Becky, was: + +"You'll not get back till late. Perhaps you'd better stay all night +with some of the girls that live near the ferry-landing, child." + +"Then I'll stay with Susy Harper, mamma." + +"Very well. And mind and behave yourself and don't be any trouble." + +Presently, as they tripped along, Tom said to Becky: + +"Say--I'll tell you what we'll do. 'Stead of going to Joe Harper's +we'll climb right up the hill and stop at the Widow Douglas'. She'll +have ice-cream! She has it most every day--dead loads of it. And she'll +be awful glad to have us." + +"Oh, that will be fun!" + +Then Becky reflected a moment and said: + +"But what will mamma say?" + +"How'll she ever know?" + +The girl turned the idea over in her mind, and said reluctantly: + +"I reckon it's wrong--but--" + +"But shucks! Your mother won't know, and so what's the harm? All she +wants is that you'll be safe; and I bet you she'd 'a' said go there if +she'd 'a' thought of it. I know she would!" + +The Widow Douglas' splendid hospitality was a tempting bait. It and +Tom's persuasions presently carried the day. So it was decided to say +nothing anybody about the night's programme. Presently it occurred to +Tom that maybe Huck might come this very night and give the signal. The +thought took a deal of the spirit out of his anticipations. Still he +could not bear to give up the fun at Widow Douglas'. And why should he +give it up, he reasoned--the signal did not come the night before, so +why should it be any more likely to come to-night? The sure fun of the +evening outweighed the uncertain treasure; and, boy-like, he determined +to yield to the stronger inclination and not allow himself to think of +the box of money another time that day. + +Three miles below town the ferryboat stopped at the mouth of a woody +hollow and tied up. The crowd swarmed ashore and soon the forest +distances and craggy heights echoed far and near with shoutings and +laughter. All the different ways of getting hot and tired were gone +through with, and by-and-by the rovers straggled back to camp fortified +with responsible appetites, and then the destruction of the good things +began. After the feast there was a refreshing season of rest and chat +in the shade of spreading oaks. By-and-by somebody shouted: + +"Who's ready for the cave?" + +Everybody was. Bundles of candles were procured, and straightway there +was a general scamper up the hill. The mouth of the cave was up the +hillside--an opening shaped like a letter A. Its massive oaken door +stood unbarred. Within was a small chamber, chilly as an ice-house, and +walled by Nature with solid limestone that was dewy with a cold sweat. +It was romantic and mysterious to stand here in the deep gloom and look +out upon the green valley shining in the sun. But the impressiveness of +the situation quickly wore off, and the romping began again. The moment +a candle was lighted there was a general rush upon the owner of it; a +struggle and a gallant defence followed, but the candle was soon +knocked down or blown out, and then there was a glad clamor of laughter +and a new chase. But all things have an end. By-and-by the procession +went filing down the steep descent of the main avenue, the flickering +rank of lights dimly revealing the lofty walls of rock almost to their +point of junction sixty feet overhead. This main avenue was not more +than eight or ten feet wide. Every few steps other lofty and still +narrower crevices branched from it on either hand--for McDougal's cave +was but a vast labyrinth of crooked aisles that ran into each other and +out again and led nowhere. It was said that one might wander days and +nights together through its intricate tangle of rifts and chasms, and +never find the end of the cave; and that he might go down, and down, +and still down, into the earth, and it was just the same--labyrinth +under labyrinth, and no end to any of them. No man "knew" the cave. +That was an impossible thing. Most of the young men knew a portion of +it, and it was not customary to venture much beyond this known portion. +Tom Sawyer knew as much of the cave as any one. + +The procession moved along the main avenue some three-quarters of a +mile, and then groups and couples began to slip aside into branch +avenues, fly along the dismal corridors, and take each other by +surprise at points where the corridors joined again. Parties were able +to elude each other for the space of half an hour without going beyond +the "known" ground. + +By-and-by, one group after another came straggling back to the mouth +of the cave, panting, hilarious, smeared from head to foot with tallow +drippings, daubed with clay, and entirely delighted with the success of +the day. Then they were astonished to find that they had been taking no +note of time and that night was about at hand. The clanging bell had +been calling for half an hour. However, this sort of close to the day's +adventures was romantic and therefore satisfactory. When the ferryboat +with her wild freight pushed into the stream, nobody cared sixpence for +the wasted time but the captain of the craft. + +Huck was already upon his watch when the ferryboat's lights went +glinting past the wharf. He heard no noise on board, for the young +people were as subdued and still as people usually are who are nearly +tired to death. He wondered what boat it was, and why she did not stop +at the wharf--and then he dropped her out of his mind and put his +attention upon his business. The night was growing cloudy and dark. Ten +o'clock came, and the noise of vehicles ceased, scattered lights began +to wink out, all straggling foot-passengers disappeared, the village +betook itself to its slumbers and left the small watcher alone with the +silence and the ghosts. Eleven o'clock came, and the tavern lights were +put out; darkness everywhere, now. Huck waited what seemed a weary long +time, but nothing happened. His faith was weakening. Was there any use? +Was there really any use? Why not give it up and turn in? + +A noise fell upon his ear. He was all attention in an instant. The +alley door closed softly. He sprang to the corner of the brick store. +The next moment two men brushed by him, and one seemed to have +something under his arm. It must be that box! So they were going to +remove the treasure. Why call Tom now? It would be absurd--the men +would get away with the box and never be found again. No, he would +stick to their wake and follow them; he would trust to the darkness for +security from discovery. So communing with himself, Huck stepped out +and glided along behind the men, cat-like, with bare feet, allowing +them to keep just far enough ahead not to be invisible. + +They moved up the river street three blocks, then turned to the left +up a cross-street. They went straight ahead, then, until they came to +the path that led up Cardiff Hill; this they took. They passed by the +old Welshman's house, half-way up the hill, without hesitating, and +still climbed upward. Good, thought Huck, they will bury it in the old +quarry. But they never stopped at the quarry. They passed on, up the +summit. They plunged into the narrow path between the tall sumach +bushes, and were at once hidden in the gloom. Huck closed up and +shortened his distance, now, for they would never be able to see him. +He trotted along awhile; then slackened his pace, fearing he was +gaining too fast; moved on a piece, then stopped altogether; listened; +no sound; none, save that he seemed to hear the beating of his own +heart. The hooting of an owl came over the hill--ominous sound! But no +footsteps. Heavens, was everything lost! He was about to spring with +winged feet, when a man cleared his throat not four feet from him! +Huck's heart shot into his throat, but he swallowed it again; and then +he stood there shaking as if a dozen agues had taken charge of him at +once, and so weak that he thought he must surely fall to the ground. He +knew where he was. He knew he was within five steps of the stile +leading into Widow Douglas' grounds. Very well, he thought, let them +bury it there; it won't be hard to find. + +Now there was a voice--a very low voice--Injun Joe's: + +"Damn her, maybe she's got company--there's lights, late as it is." + +"I can't see any." + +This was that stranger's voice--the stranger of the haunted house. A +deadly chill went to Huck's heart--this, then, was the "revenge" job! +His thought was, to fly. Then he remembered that the Widow Douglas had +been kind to him more than once, and maybe these men were going to +murder her. He wished he dared venture to warn her; but he knew he +didn't dare--they might come and catch him. He thought all this and +more in the moment that elapsed between the stranger's remark and Injun +Joe's next--which was-- + +"Because the bush is in your way. Now--this way--now you see, don't +you?" + +"Yes. Well, there IS company there, I reckon. Better give it up." + +"Give it up, and I just leaving this country forever! Give it up and +maybe never have another chance. I tell you again, as I've told you +before, I don't care for her swag--you may have it. But her husband was +rough on me--many times he was rough on me--and mainly he was the +justice of the peace that jugged me for a vagrant. And that ain't all. +It ain't a millionth part of it! He had me HORSEWHIPPED!--horsewhipped +in front of the jail, like a nigger!--with all the town looking on! +HORSEWHIPPED!--do you understand? He took advantage of me and died. But +I'll take it out of HER." + +"Oh, don't kill her! Don't do that!" + +"Kill? Who said anything about killing? I would kill HIM if he was +here; but not her. When you want to get revenge on a woman you don't +kill her--bosh! you go for her looks. You slit her nostrils--you notch +her ears like a sow!" + +"By God, that's--" + +"Keep your opinion to yourself! It will be safest for you. I'll tie +her to the bed. If she bleeds to death, is that my fault? I'll not cry, +if she does. My friend, you'll help me in this thing--for MY sake +--that's why you're here--I mightn't be able alone. If you flinch, I'll +kill you. Do you understand that? And if I have to kill you, I'll kill +her--and then I reckon nobody'll ever know much about who done this +business." + +"Well, if it's got to be done, let's get at it. The quicker the +better--I'm all in a shiver." + +"Do it NOW? And company there? Look here--I'll get suspicious of you, +first thing you know. No--we'll wait till the lights are out--there's +no hurry." + +Huck felt that a silence was going to ensue--a thing still more awful +than any amount of murderous talk; so he held his breath and stepped +gingerly back; planted his foot carefully and firmly, after balancing, +one-legged, in a precarious way and almost toppling over, first on one +side and then on the other. He took another step back, with the same +elaboration and the same risks; then another and another, and--a twig +snapped under his foot! His breath stopped and he listened. There was +no sound--the stillness was perfect. His gratitude was measureless. Now +he turned in his tracks, between the walls of sumach bushes--turned +himself as carefully as if he were a ship--and then stepped quickly but +cautiously along. When he emerged at the quarry he felt secure, and so +he picked up his nimble heels and flew. Down, down he sped, till he +reached the Welshman's. He banged at the door, and presently the heads +of the old man and his two stalwart sons were thrust from windows. + +"What's the row there? Who's banging? What do you want?" + +"Let me in--quick! I'll tell everything." + +"Why, who are you?" + +"Huckleberry Finn--quick, let me in!" + +"Huckleberry Finn, indeed! It ain't a name to open many doors, I +judge! But let him in, lads, and let's see what's the trouble." + +"Please don't ever tell I told you," were Huck's first words when he +got in. "Please don't--I'd be killed, sure--but the widow's been good +friends to me sometimes, and I want to tell--I WILL tell if you'll +promise you won't ever say it was me." + +"By George, he HAS got something to tell, or he wouldn't act so!" +exclaimed the old man; "out with it and nobody here'll ever tell, lad." + +Three minutes later the old man and his sons, well armed, were up the +hill, and just entering the sumach path on tiptoe, their weapons in +their hands. Huck accompanied them no further. He hid behind a great +bowlder and fell to listening. There was a lagging, anxious silence, +and then all of a sudden there was an explosion of firearms and a cry. + +Huck waited for no particulars. He sprang away and sped down the hill +as fast as his legs could carry him. + + + +CHAPTER XXX + +AS the earliest suspicion of dawn appeared on Sunday morning, Huck +came groping up the hill and rapped gently at the old Welshman's door. +The inmates were asleep, but it was a sleep that was set on a +hair-trigger, on account of the exciting episode of the night. A call +came from a window: + +"Who's there!" + +Huck's scared voice answered in a low tone: + +"Please let me in! It's only Huck Finn!" + +"It's a name that can open this door night or day, lad!--and welcome!" + +These were strange words to the vagabond boy's ears, and the +pleasantest he had ever heard. He could not recollect that the closing +word had ever been applied in his case before. The door was quickly +unlocked, and he entered. Huck was given a seat and the old man and his +brace of tall sons speedily dressed themselves. + +"Now, my boy, I hope you're good and hungry, because breakfast will be +ready as soon as the sun's up, and we'll have a piping hot one, too +--make yourself easy about that! I and the boys hoped you'd turn up and +stop here last night." + +"I was awful scared," said Huck, "and I run. I took out when the +pistols went off, and I didn't stop for three mile. I've come now becuz +I wanted to know about it, you know; and I come before daylight becuz I +didn't want to run across them devils, even if they was dead." + +"Well, poor chap, you do look as if you'd had a hard night of it--but +there's a bed here for you when you've had your breakfast. No, they +ain't dead, lad--we are sorry enough for that. You see we knew right +where to put our hands on them, by your description; so we crept along +on tiptoe till we got within fifteen feet of them--dark as a cellar +that sumach path was--and just then I found I was going to sneeze. It +was the meanest kind of luck! I tried to keep it back, but no use +--'twas bound to come, and it did come! I was in the lead with my pistol +raised, and when the sneeze started those scoundrels a-rustling to get +out of the path, I sung out, 'Fire boys!' and blazed away at the place +where the rustling was. So did the boys. But they were off in a jiffy, +those villains, and we after them, down through the woods. I judge we +never touched them. They fired a shot apiece as they started, but their +bullets whizzed by and didn't do us any harm. As soon as we lost the +sound of their feet we quit chasing, and went down and stirred up the +constables. They got a posse together, and went off to guard the river +bank, and as soon as it is light the sheriff and a gang are going to +beat up the woods. My boys will be with them presently. I wish we had +some sort of description of those rascals--'twould help a good deal. +But you couldn't see what they were like, in the dark, lad, I suppose?" + +"Oh yes; I saw them down-town and follered them." + +"Splendid! Describe them--describe them, my boy!" + +"One's the old deaf and dumb Spaniard that's ben around here once or +twice, and t'other's a mean-looking, ragged--" + +"That's enough, lad, we know the men! Happened on them in the woods +back of the widow's one day, and they slunk away. Off with you, boys, +and tell the sheriff--get your breakfast to-morrow morning!" + +The Welshman's sons departed at once. As they were leaving the room +Huck sprang up and exclaimed: + +"Oh, please don't tell ANYbody it was me that blowed on them! Oh, +please!" + +"All right if you say it, Huck, but you ought to have the credit of +what you did." + +"Oh no, no! Please don't tell!" + +When the young men were gone, the old Welshman said: + +"They won't tell--and I won't. But why don't you want it known?" + +Huck would not explain, further than to say that he already knew too +much about one of those men and would not have the man know that he +knew anything against him for the whole world--he would be killed for +knowing it, sure. + +The old man promised secrecy once more, and said: + +"How did you come to follow these fellows, lad? Were they looking +suspicious?" + +Huck was silent while he framed a duly cautious reply. Then he said: + +"Well, you see, I'm a kind of a hard lot,--least everybody says so, +and I don't see nothing agin it--and sometimes I can't sleep much, on +account of thinking about it and sort of trying to strike out a new way +of doing. That was the way of it last night. I couldn't sleep, and so I +come along up-street 'bout midnight, a-turning it all over, and when I +got to that old shackly brick store by the Temperance Tavern, I backed +up agin the wall to have another think. Well, just then along comes +these two chaps slipping along close by me, with something under their +arm, and I reckoned they'd stole it. One was a-smoking, and t'other one +wanted a light; so they stopped right before me and the cigars lit up +their faces and I see that the big one was the deaf and dumb Spaniard, +by his white whiskers and the patch on his eye, and t'other one was a +rusty, ragged-looking devil." + +"Could you see the rags by the light of the cigars?" + +This staggered Huck for a moment. Then he said: + +"Well, I don't know--but somehow it seems as if I did." + +"Then they went on, and you--" + +"Follered 'em--yes. That was it. I wanted to see what was up--they +sneaked along so. I dogged 'em to the widder's stile, and stood in the +dark and heard the ragged one beg for the widder, and the Spaniard +swear he'd spile her looks just as I told you and your two--" + +"What! The DEAF AND DUMB man said all that!" + +Huck had made another terrible mistake! He was trying his best to keep +the old man from getting the faintest hint of who the Spaniard might +be, and yet his tongue seemed determined to get him into trouble in +spite of all he could do. He made several efforts to creep out of his +scrape, but the old man's eye was upon him and he made blunder after +blunder. Presently the Welshman said: + +"My boy, don't be afraid of me. I wouldn't hurt a hair of your head +for all the world. No--I'd protect you--I'd protect you. This Spaniard +is not deaf and dumb; you've let that slip without intending it; you +can't cover that up now. You know something about that Spaniard that +you want to keep dark. Now trust me--tell me what it is, and trust me +--I won't betray you." + +Huck looked into the old man's honest eyes a moment, then bent over +and whispered in his ear: + +"'Tain't a Spaniard--it's Injun Joe!" + +The Welshman almost jumped out of his chair. In a moment he said: + +"It's all plain enough, now. When you talked about notching ears and +slitting noses I judged that that was your own embellishment, because +white men don't take that sort of revenge. But an Injun! That's a +different matter altogether." + +During breakfast the talk went on, and in the course of it the old man +said that the last thing which he and his sons had done, before going +to bed, was to get a lantern and examine the stile and its vicinity for +marks of blood. They found none, but captured a bulky bundle of-- + +"Of WHAT?" + +If the words had been lightning they could not have leaped with a more +stunning suddenness from Huck's blanched lips. His eyes were staring +wide, now, and his breath suspended--waiting for the answer. The +Welshman started--stared in return--three seconds--five seconds--ten +--then replied: + +"Of burglar's tools. Why, what's the MATTER with you?" + +Huck sank back, panting gently, but deeply, unutterably grateful. The +Welshman eyed him gravely, curiously--and presently said: + +"Yes, burglar's tools. That appears to relieve you a good deal. But +what did give you that turn? What were YOU expecting we'd found?" + +Huck was in a close place--the inquiring eye was upon him--he would +have given anything for material for a plausible answer--nothing +suggested itself--the inquiring eye was boring deeper and deeper--a +senseless reply offered--there was no time to weigh it, so at a venture +he uttered it--feebly: + +"Sunday-school books, maybe." + +Poor Huck was too distressed to smile, but the old man laughed loud +and joyously, shook up the details of his anatomy from head to foot, +and ended by saying that such a laugh was money in a-man's pocket, +because it cut down the doctor's bill like everything. Then he added: + +"Poor old chap, you're white and jaded--you ain't well a bit--no +wonder you're a little flighty and off your balance. But you'll come +out of it. Rest and sleep will fetch you out all right, I hope." + +Huck was irritated to think he had been such a goose and betrayed such +a suspicious excitement, for he had dropped the idea that the parcel +brought from the tavern was the treasure, as soon as he had heard the +talk at the widow's stile. He had only thought it was not the treasure, +however--he had not known that it wasn't--and so the suggestion of a +captured bundle was too much for his self-possession. But on the whole +he felt glad the little episode had happened, for now he knew beyond +all question that that bundle was not THE bundle, and so his mind was +at rest and exceedingly comfortable. In fact, everything seemed to be +drifting just in the right direction, now; the treasure must be still +in No. 2, the men would be captured and jailed that day, and he and Tom +could seize the gold that night without any trouble or any fear of +interruption. + +Just as breakfast was completed there was a knock at the door. Huck +jumped for a hiding-place, for he had no mind to be connected even +remotely with the late event. The Welshman admitted several ladies and +gentlemen, among them the Widow Douglas, and noticed that groups of +citizens were climbing up the hill--to stare at the stile. So the news +had spread. The Welshman had to tell the story of the night to the +visitors. The widow's gratitude for her preservation was outspoken. + +"Don't say a word about it, madam. There's another that you're more +beholden to than you are to me and my boys, maybe, but he don't allow +me to tell his name. We wouldn't have been there but for him." + +Of course this excited a curiosity so vast that it almost belittled +the main matter--but the Welshman allowed it to eat into the vitals of +his visitors, and through them be transmitted to the whole town, for he +refused to part with his secret. When all else had been learned, the +widow said: + +"I went to sleep reading in bed and slept straight through all that +noise. Why didn't you come and wake me?" + +"We judged it warn't worth while. Those fellows warn't likely to come +again--they hadn't any tools left to work with, and what was the use of +waking you up and scaring you to death? My three negro men stood guard +at your house all the rest of the night. They've just come back." + +More visitors came, and the story had to be told and retold for a +couple of hours more. + +There was no Sabbath-school during day-school vacation, but everybody +was early at church. The stirring event was well canvassed. News came +that not a sign of the two villains had been yet discovered. When the +sermon was finished, Judge Thatcher's wife dropped alongside of Mrs. +Harper as she moved down the aisle with the crowd and said: + +"Is my Becky going to sleep all day? I just expected she would be +tired to death." + +"Your Becky?" + +"Yes," with a startled look--"didn't she stay with you last night?" + +"Why, no." + +Mrs. Thatcher turned pale, and sank into a pew, just as Aunt Polly, +talking briskly with a friend, passed by. Aunt Polly said: + +"Good-morning, Mrs. Thatcher. Good-morning, Mrs. Harper. I've got a +boy that's turned up missing. I reckon my Tom stayed at your house last +night--one of you. And now he's afraid to come to church. I've got to +settle with him." + +Mrs. Thatcher shook her head feebly and turned paler than ever. + +"He didn't stay with us," said Mrs. Harper, beginning to look uneasy. +A marked anxiety came into Aunt Polly's face. + +"Joe Harper, have you seen my Tom this morning?" + +"No'm." + +"When did you see him last?" + +Joe tried to remember, but was not sure he could say. The people had +stopped moving out of church. Whispers passed along, and a boding +uneasiness took possession of every countenance. Children were +anxiously questioned, and young teachers. They all said they had not +noticed whether Tom and Becky were on board the ferryboat on the +homeward trip; it was dark; no one thought of inquiring if any one was +missing. One young man finally blurted out his fear that they were +still in the cave! Mrs. Thatcher swooned away. Aunt Polly fell to +crying and wringing her hands. + +The alarm swept from lip to lip, from group to group, from street to +street, and within five minutes the bells were wildly clanging and the +whole town was up! The Cardiff Hill episode sank into instant +insignificance, the burglars were forgotten, horses were saddled, +skiffs were manned, the ferryboat ordered out, and before the horror +was half an hour old, two hundred men were pouring down highroad and +river toward the cave. + +All the long afternoon the village seemed empty and dead. Many women +visited Aunt Polly and Mrs. Thatcher and tried to comfort them. They +cried with them, too, and that was still better than words. All the +tedious night the town waited for news; but when the morning dawned at +last, all the word that came was, "Send more candles--and send food." +Mrs. Thatcher was almost crazed; and Aunt Polly, also. Judge Thatcher +sent messages of hope and encouragement from the cave, but they +conveyed no real cheer. + +The old Welshman came home toward daylight, spattered with +candle-grease, smeared with clay, and almost worn out. He found Huck +still in the bed that had been provided for him, and delirious with +fever. The physicians were all at the cave, so the Widow Douglas came +and took charge of the patient. She said she would do her best by him, +because, whether he was good, bad, or indifferent, he was the Lord's, +and nothing that was the Lord's was a thing to be neglected. The +Welshman said Huck had good spots in him, and the widow said: + +"You can depend on it. That's the Lord's mark. He don't leave it off. +He never does. Puts it somewhere on every creature that comes from his +hands." + +Early in the forenoon parties of jaded men began to straggle into the +village, but the strongest of the citizens continued searching. All the +news that could be gained was that remotenesses of the cavern were +being ransacked that had never been visited before; that every corner +and crevice was going to be thoroughly searched; that wherever one +wandered through the maze of passages, lights were to be seen flitting +hither and thither in the distance, and shoutings and pistol-shots sent +their hollow reverberations to the ear down the sombre aisles. In one +place, far from the section usually traversed by tourists, the names +"BECKY & TOM" had been found traced upon the rocky wall with +candle-smoke, and near at hand a grease-soiled bit of ribbon. Mrs. +Thatcher recognized the ribbon and cried over it. She said it was the +last relic she should ever have of her child; and that no other memorial +of her could ever be so precious, because this one parted latest from +the living body before the awful death came. Some said that now and +then, in the cave, a far-away speck of light would glimmer, and then a +glorious shout would burst forth and a score of men go trooping down the +echoing aisle--and then a sickening disappointment always followed; the +children were not there; it was only a searcher's light. + +Three dreadful days and nights dragged their tedious hours along, and +the village sank into a hopeless stupor. No one had heart for anything. +The accidental discovery, just made, that the proprietor of the +Temperance Tavern kept liquor on his premises, scarcely fluttered the +public pulse, tremendous as the fact was. In a lucid interval, Huck +feebly led up to the subject of taverns, and finally asked--dimly +dreading the worst--if anything had been discovered at the Temperance +Tavern since he had been ill. + +"Yes," said the widow. + +Huck started up in bed, wild-eyed: + +"What? What was it?" + +"Liquor!--and the place has been shut up. Lie down, child--what a turn +you did give me!" + +"Only tell me just one thing--only just one--please! Was it Tom Sawyer +that found it?" + +The widow burst into tears. "Hush, hush, child, hush! I've told you +before, you must NOT talk. You are very, very sick!" + +Then nothing but liquor had been found; there would have been a great +powwow if it had been the gold. So the treasure was gone forever--gone +forever! But what could she be crying about? Curious that she should +cry. + +These thoughts worked their dim way through Huck's mind, and under the +weariness they gave him he fell asleep. The widow said to herself: + +"There--he's asleep, poor wreck. Tom Sawyer find it! Pity but somebody +could find Tom Sawyer! Ah, there ain't many left, now, that's got hope +enough, or strength enough, either, to go on searching." + + + +CHAPTER XXXI + +NOW to return to Tom and Becky's share in the picnic. They tripped +along the murky aisles with the rest of the company, visiting the +familiar wonders of the cave--wonders dubbed with rather +over-descriptive names, such as "The Drawing-Room," "The Cathedral," +"Aladdin's Palace," and so on. Presently the hide-and-seek frolicking +began, and Tom and Becky engaged in it with zeal until the exertion +began to grow a trifle wearisome; then they wandered down a sinuous +avenue holding their candles aloft and reading the tangled web-work of +names, dates, post-office addresses, and mottoes with which the rocky +walls had been frescoed (in candle-smoke). Still drifting along and +talking, they scarcely noticed that they were now in a part of the cave +whose walls were not frescoed. They smoked their own names under an +overhanging shelf and moved on. Presently they came to a place where a +little stream of water, trickling over a ledge and carrying a limestone +sediment with it, had, in the slow-dragging ages, formed a laced and +ruffled Niagara in gleaming and imperishable stone. Tom squeezed his +small body behind it in order to illuminate it for Becky's +gratification. He found that it curtained a sort of steep natural +stairway which was enclosed between narrow walls, and at once the +ambition to be a discoverer seized him. Becky responded to his call, +and they made a smoke-mark for future guidance, and started upon their +quest. They wound this way and that, far down into the secret depths of +the cave, made another mark, and branched off in search of novelties to +tell the upper world about. In one place they found a spacious cavern, +from whose ceiling depended a multitude of shining stalactites of the +length and circumference of a man's leg; they walked all about it, +wondering and admiring, and presently left it by one of the numerous +passages that opened into it. This shortly brought them to a bewitching +spring, whose basin was incrusted with a frostwork of glittering +crystals; it was in the midst of a cavern whose walls were supported by +many fantastic pillars which had been formed by the joining of great +stalactites and stalagmites together, the result of the ceaseless +water-drip of centuries. Under the roof vast knots of bats had packed +themselves together, thousands in a bunch; the lights disturbed the +creatures and they came flocking down by hundreds, squeaking and +darting furiously at the candles. Tom knew their ways and the danger of +this sort of conduct. He seized Becky's hand and hurried her into the +first corridor that offered; and none too soon, for a bat struck +Becky's light out with its wing while she was passing out of the +cavern. The bats chased the children a good distance; but the fugitives +plunged into every new passage that offered, and at last got rid of the +perilous things. Tom found a subterranean lake, shortly, which +stretched its dim length away until its shape was lost in the shadows. +He wanted to explore its borders, but concluded that it would be best +to sit down and rest awhile, first. Now, for the first time, the deep +stillness of the place laid a clammy hand upon the spirits of the +children. Becky said: + +"Why, I didn't notice, but it seems ever so long since I heard any of +the others." + +"Come to think, Becky, we are away down below them--and I don't know +how far away north, or south, or east, or whichever it is. We couldn't +hear them here." + +Becky grew apprehensive. + +"I wonder how long we've been down here, Tom? We better start back." + +"Yes, I reckon we better. P'raps we better." + +"Can you find the way, Tom? It's all a mixed-up crookedness to me." + +"I reckon I could find it--but then the bats. If they put our candles +out it will be an awful fix. Let's try some other way, so as not to go +through there." + +"Well. But I hope we won't get lost. It would be so awful!" and the +girl shuddered at the thought of the dreadful possibilities. + +They started through a corridor, and traversed it in silence a long +way, glancing at each new opening, to see if there was anything +familiar about the look of it; but they were all strange. Every time +Tom made an examination, Becky would watch his face for an encouraging +sign, and he would say cheerily: + +"Oh, it's all right. This ain't the one, but we'll come to it right +away!" + +But he felt less and less hopeful with each failure, and presently +began to turn off into diverging avenues at sheer random, in desperate +hope of finding the one that was wanted. He still said it was "all +right," but there was such a leaden dread at his heart that the words +had lost their ring and sounded just as if he had said, "All is lost!" +Becky clung to his side in an anguish of fear, and tried hard to keep +back the tears, but they would come. At last she said: + +"Oh, Tom, never mind the bats, let's go back that way! We seem to get +worse and worse off all the time." + +"Listen!" said he. + +Profound silence; silence so deep that even their breathings were +conspicuous in the hush. Tom shouted. The call went echoing down the +empty aisles and died out in the distance in a faint sound that +resembled a ripple of mocking laughter. + +"Oh, don't do it again, Tom, it is too horrid," said Becky. + +"It is horrid, but I better, Becky; they might hear us, you know," and +he shouted again. + +The "might" was even a chillier horror than the ghostly laughter, it +so confessed a perishing hope. The children stood still and listened; +but there was no result. Tom turned upon the back track at once, and +hurried his steps. It was but a little while before a certain +indecision in his manner revealed another fearful fact to Becky--he +could not find his way back! + +"Oh, Tom, you didn't make any marks!" + +"Becky, I was such a fool! Such a fool! I never thought we might want +to come back! No--I can't find the way. It's all mixed up." + +"Tom, Tom, we're lost! we're lost! We never can get out of this awful +place! Oh, why DID we ever leave the others!" + +She sank to the ground and burst into such a frenzy of crying that Tom +was appalled with the idea that she might die, or lose her reason. He +sat down by her and put his arms around her; she buried her face in his +bosom, she clung to him, she poured out her terrors, her unavailing +regrets, and the far echoes turned them all to jeering laughter. Tom +begged her to pluck up hope again, and she said she could not. He fell +to blaming and abusing himself for getting her into this miserable +situation; this had a better effect. She said she would try to hope +again, she would get up and follow wherever he might lead if only he +would not talk like that any more. For he was no more to blame than +she, she said. + +So they moved on again--aimlessly--simply at random--all they could do +was to move, keep moving. For a little while, hope made a show of +reviving--not with any reason to back it, but only because it is its +nature to revive when the spring has not been taken out of it by age +and familiarity with failure. + +By-and-by Tom took Becky's candle and blew it out. This economy meant +so much! Words were not needed. Becky understood, and her hope died +again. She knew that Tom had a whole candle and three or four pieces in +his pockets--yet he must economize. + +By-and-by, fatigue began to assert its claims; the children tried to +pay attention, for it was dreadful to think of sitting down when time +was grown to be so precious, moving, in some direction, in any +direction, was at least progress and might bear fruit; but to sit down +was to invite death and shorten its pursuit. + +At last Becky's frail limbs refused to carry her farther. She sat +down. Tom rested with her, and they talked of home, and the friends +there, and the comfortable beds and, above all, the light! Becky cried, +and Tom tried to think of some way of comforting her, but all his +encouragements were grown threadbare with use, and sounded like +sarcasms. Fatigue bore so heavily upon Becky that she drowsed off to +sleep. Tom was grateful. He sat looking into her drawn face and saw it +grow smooth and natural under the influence of pleasant dreams; and +by-and-by a smile dawned and rested there. The peaceful face reflected +somewhat of peace and healing into his own spirit, and his thoughts +wandered away to bygone times and dreamy memories. While he was deep in +his musings, Becky woke up with a breezy little laugh--but it was +stricken dead upon her lips, and a groan followed it. + +"Oh, how COULD I sleep! I wish I never, never had waked! No! No, I +don't, Tom! Don't look so! I won't say it again." + +"I'm glad you've slept, Becky; you'll feel rested, now, and we'll find +the way out." + +"We can try, Tom; but I've seen such a beautiful country in my dream. +I reckon we are going there." + +"Maybe not, maybe not. Cheer up, Becky, and let's go on trying." + +They rose up and wandered along, hand in hand and hopeless. They tried +to estimate how long they had been in the cave, but all they knew was +that it seemed days and weeks, and yet it was plain that this could not +be, for their candles were not gone yet. A long time after this--they +could not tell how long--Tom said they must go softly and listen for +dripping water--they must find a spring. They found one presently, and +Tom said it was time to rest again. Both were cruelly tired, yet Becky +said she thought she could go a little farther. She was surprised to +hear Tom dissent. She could not understand it. They sat down, and Tom +fastened his candle to the wall in front of them with some clay. +Thought was soon busy; nothing was said for some time. Then Becky broke +the silence: + +"Tom, I am so hungry!" + +Tom took something out of his pocket. + +"Do you remember this?" said he. + +Becky almost smiled. + +"It's our wedding-cake, Tom." + +"Yes--I wish it was as big as a barrel, for it's all we've got." + +"I saved it from the picnic for us to dream on, Tom, the way grown-up +people do with wedding-cake--but it'll be our--" + +She dropped the sentence where it was. Tom divided the cake and Becky +ate with good appetite, while Tom nibbled at his moiety. There was +abundance of cold water to finish the feast with. By-and-by Becky +suggested that they move on again. Tom was silent a moment. Then he +said: + +"Becky, can you bear it if I tell you something?" + +Becky's face paled, but she thought she could. + +"Well, then, Becky, we must stay here, where there's water to drink. +That little piece is our last candle!" + +Becky gave loose to tears and wailings. Tom did what he could to +comfort her, but with little effect. At length Becky said: + +"Tom!" + +"Well, Becky?" + +"They'll miss us and hunt for us!" + +"Yes, they will! Certainly they will!" + +"Maybe they're hunting for us now, Tom." + +"Why, I reckon maybe they are. I hope they are." + +"When would they miss us, Tom?" + +"When they get back to the boat, I reckon." + +"Tom, it might be dark then--would they notice we hadn't come?" + +"I don't know. But anyway, your mother would miss you as soon as they +got home." + +A frightened look in Becky's face brought Tom to his senses and he saw +that he had made a blunder. Becky was not to have gone home that night! +The children became silent and thoughtful. In a moment a new burst of +grief from Becky showed Tom that the thing in his mind had struck hers +also--that the Sabbath morning might be half spent before Mrs. Thatcher +discovered that Becky was not at Mrs. Harper's. + +The children fastened their eyes upon their bit of candle and watched +it melt slowly and pitilessly away; saw the half inch of wick stand +alone at last; saw the feeble flame rise and fall, climb the thin +column of smoke, linger at its top a moment, and then--the horror of +utter darkness reigned! + +How long afterward it was that Becky came to a slow consciousness that +she was crying in Tom's arms, neither could tell. All that they knew +was, that after what seemed a mighty stretch of time, both awoke out of +a dead stupor of sleep and resumed their miseries once more. Tom said +it might be Sunday, now--maybe Monday. He tried to get Becky to talk, +but her sorrows were too oppressive, all her hopes were gone. Tom said +that they must have been missed long ago, and no doubt the search was +going on. He would shout and maybe some one would come. He tried it; +but in the darkness the distant echoes sounded so hideously that he +tried it no more. + +The hours wasted away, and hunger came to torment the captives again. +A portion of Tom's half of the cake was left; they divided and ate it. +But they seemed hungrier than before. The poor morsel of food only +whetted desire. + +By-and-by Tom said: + +"SH! Did you hear that?" + +Both held their breath and listened. There was a sound like the +faintest, far-off shout. Instantly Tom answered it, and leading Becky +by the hand, started groping down the corridor in its direction. +Presently he listened again; again the sound was heard, and apparently +a little nearer. + +"It's them!" said Tom; "they're coming! Come along, Becky--we're all +right now!" + +The joy of the prisoners was almost overwhelming. Their speed was +slow, however, because pitfalls were somewhat common, and had to be +guarded against. They shortly came to one and had to stop. It might be +three feet deep, it might be a hundred--there was no passing it at any +rate. Tom got down on his breast and reached as far down as he could. +No bottom. They must stay there and wait until the searchers came. They +listened; evidently the distant shoutings were growing more distant! a +moment or two more and they had gone altogether. The heart-sinking +misery of it! Tom whooped until he was hoarse, but it was of no use. He +talked hopefully to Becky; but an age of anxious waiting passed and no +sounds came again. + +The children groped their way back to the spring. The weary time +dragged on; they slept again, and awoke famished and woe-stricken. Tom +believed it must be Tuesday by this time. + +Now an idea struck him. There were some side passages near at hand. It +would be better to explore some of these than bear the weight of the +heavy time in idleness. He took a kite-line from his pocket, tied it to +a projection, and he and Becky started, Tom in the lead, unwinding the +line as he groped along. At the end of twenty steps the corridor ended +in a "jumping-off place." Tom got down on his knees and felt below, and +then as far around the corner as he could reach with his hands +conveniently; he made an effort to stretch yet a little farther to the +right, and at that moment, not twenty yards away, a human hand, holding +a candle, appeared from behind a rock! Tom lifted up a glorious shout, +and instantly that hand was followed by the body it belonged to--Injun +Joe's! Tom was paralyzed; he could not move. He was vastly gratified +the next moment, to see the "Spaniard" take to his heels and get +himself out of sight. Tom wondered that Joe had not recognized his +voice and come over and killed him for testifying in court. But the +echoes must have disguised the voice. Without doubt, that was it, he +reasoned. Tom's fright weakened every muscle in his body. He said to +himself that if he had strength enough to get back to the spring he +would stay there, and nothing should tempt him to run the risk of +meeting Injun Joe again. He was careful to keep from Becky what it was +he had seen. He told her he had only shouted "for luck." + +But hunger and wretchedness rise superior to fears in the long run. +Another tedious wait at the spring and another long sleep brought +changes. The children awoke tortured with a raging hunger. Tom believed +that it must be Wednesday or Thursday or even Friday or Saturday, now, +and that the search had been given over. He proposed to explore another +passage. He felt willing to risk Injun Joe and all other terrors. But +Becky was very weak. She had sunk into a dreary apathy and would not be +roused. She said she would wait, now, where she was, and die--it would +not be long. She told Tom to go with the kite-line and explore if he +chose; but she implored him to come back every little while and speak +to her; and she made him promise that when the awful time came, he +would stay by her and hold her hand until all was over. + +Tom kissed her, with a choking sensation in his throat, and made a +show of being confident of finding the searchers or an escape from the +cave; then he took the kite-line in his hand and went groping down one +of the passages on his hands and knees, distressed with hunger and sick +with bodings of coming doom. + + + +CHAPTER XXXII + +TUESDAY afternoon came, and waned to the twilight. The village of St. +Petersburg still mourned. The lost children had not been found. Public +prayers had been offered up for them, and many and many a private +prayer that had the petitioner's whole heart in it; but still no good +news came from the cave. The majority of the searchers had given up the +quest and gone back to their daily avocations, saying that it was plain +the children could never be found. Mrs. Thatcher was very ill, and a +great part of the time delirious. People said it was heartbreaking to +hear her call her child, and raise her head and listen a whole minute +at a time, then lay it wearily down again with a moan. Aunt Polly had +drooped into a settled melancholy, and her gray hair had grown almost +white. The village went to its rest on Tuesday night, sad and forlorn. + +Away in the middle of the night a wild peal burst from the village +bells, and in a moment the streets were swarming with frantic half-clad +people, who shouted, "Turn out! turn out! they're found! they're +found!" Tin pans and horns were added to the din, the population massed +itself and moved toward the river, met the children coming in an open +carriage drawn by shouting citizens, thronged around it, joined its +homeward march, and swept magnificently up the main street roaring +huzzah after huzzah! + +The village was illuminated; nobody went to bed again; it was the +greatest night the little town had ever seen. During the first half-hour +a procession of villagers filed through Judge Thatcher's house, seized +the saved ones and kissed them, squeezed Mrs. Thatcher's hand, tried to +speak but couldn't--and drifted out raining tears all over the place. + +Aunt Polly's happiness was complete, and Mrs. Thatcher's nearly so. It +would be complete, however, as soon as the messenger dispatched with +the great news to the cave should get the word to her husband. Tom lay +upon a sofa with an eager auditory about him and told the history of +the wonderful adventure, putting in many striking additions to adorn it +withal; and closed with a description of how he left Becky and went on +an exploring expedition; how he followed two avenues as far as his +kite-line would reach; how he followed a third to the fullest stretch of +the kite-line, and was about to turn back when he glimpsed a far-off +speck that looked like daylight; dropped the line and groped toward it, +pushed his head and shoulders through a small hole, and saw the broad +Mississippi rolling by! And if it had only happened to be night he would +not have seen that speck of daylight and would not have explored that +passage any more! He told how he went back for Becky and broke the good +news and she told him not to fret her with such stuff, for she was +tired, and knew she was going to die, and wanted to. He described how he +labored with her and convinced her; and how she almost died for joy when +she had groped to where she actually saw the blue speck of daylight; how +he pushed his way out at the hole and then helped her out; how they sat +there and cried for gladness; how some men came along in a skiff and Tom +hailed them and told them their situation and their famished condition; +how the men didn't believe the wild tale at first, "because," said they, +"you are five miles down the river below the valley the cave is in" +--then took them aboard, rowed to a house, gave them supper, made them +rest till two or three hours after dark and then brought them home. + +Before day-dawn, Judge Thatcher and the handful of searchers with him +were tracked out, in the cave, by the twine clews they had strung +behind them, and informed of the great news. + +Three days and nights of toil and hunger in the cave were not to be +shaken off at once, as Tom and Becky soon discovered. They were +bedridden all of Wednesday and Thursday, and seemed to grow more and +more tired and worn, all the time. Tom got about, a little, on +Thursday, was down-town Friday, and nearly as whole as ever Saturday; +but Becky did not leave her room until Sunday, and then she looked as +if she had passed through a wasting illness. + +Tom learned of Huck's sickness and went to see him on Friday, but +could not be admitted to the bedroom; neither could he on Saturday or +Sunday. He was admitted daily after that, but was warned to keep still +about his adventure and introduce no exciting topic. The Widow Douglas +stayed by to see that he obeyed. At home Tom learned of the Cardiff +Hill event; also that the "ragged man's" body had eventually been found +in the river near the ferry-landing; he had been drowned while trying +to escape, perhaps. + +About a fortnight after Tom's rescue from the cave, he started off to +visit Huck, who had grown plenty strong enough, now, to hear exciting +talk, and Tom had some that would interest him, he thought. Judge +Thatcher's house was on Tom's way, and he stopped to see Becky. The +Judge and some friends set Tom to talking, and some one asked him +ironically if he wouldn't like to go to the cave again. Tom said he +thought he wouldn't mind it. The Judge said: + +"Well, there are others just like you, Tom, I've not the least doubt. +But we have taken care of that. Nobody will get lost in that cave any +more." + +"Why?" + +"Because I had its big door sheathed with boiler iron two weeks ago, +and triple-locked--and I've got the keys." + +Tom turned as white as a sheet. + +"What's the matter, boy! Here, run, somebody! Fetch a glass of water!" + +The water was brought and thrown into Tom's face. + +"Ah, now you're all right. What was the matter with you, Tom?" + +"Oh, Judge, Injun Joe's in the cave!" + + + +CHAPTER XXXIII + +WITHIN a few minutes the news had spread, and a dozen skiff-loads of +men were on their way to McDougal's cave, and the ferryboat, well +filled with passengers, soon followed. Tom Sawyer was in the skiff that +bore Judge Thatcher. + +When the cave door was unlocked, a sorrowful sight presented itself in +the dim twilight of the place. Injun Joe lay stretched upon the ground, +dead, with his face close to the crack of the door, as if his longing +eyes had been fixed, to the latest moment, upon the light and the cheer +of the free world outside. Tom was touched, for he knew by his own +experience how this wretch had suffered. His pity was moved, but +nevertheless he felt an abounding sense of relief and security, now, +which revealed to him in a degree which he had not fully appreciated +before how vast a weight of dread had been lying upon him since the day +he lifted his voice against this bloody-minded outcast. + +Injun Joe's bowie-knife lay close by, its blade broken in two. The +great foundation-beam of the door had been chipped and hacked through, +with tedious labor; useless labor, too, it was, for the native rock +formed a sill outside it, and upon that stubborn material the knife had +wrought no effect; the only damage done was to the knife itself. But if +there had been no stony obstruction there the labor would have been +useless still, for if the beam had been wholly cut away Injun Joe could +not have squeezed his body under the door, and he knew it. So he had +only hacked that place in order to be doing something--in order to pass +the weary time--in order to employ his tortured faculties. Ordinarily +one could find half a dozen bits of candle stuck around in the crevices +of this vestibule, left there by tourists; but there were none now. The +prisoner had searched them out and eaten them. He had also contrived to +catch a few bats, and these, also, he had eaten, leaving only their +claws. The poor unfortunate had starved to death. In one place, near at +hand, a stalagmite had been slowly growing up from the ground for ages, +builded by the water-drip from a stalactite overhead. The captive had +broken off the stalagmite, and upon the stump had placed a stone, +wherein he had scooped a shallow hollow to catch the precious drop +that fell once in every three minutes with the dreary regularity of a +clock-tick--a dessertspoonful once in four and twenty hours. That drop +was falling when the Pyramids were new; when Troy fell; when the +foundations of Rome were laid; when Christ was crucified; when the +Conqueror created the British empire; when Columbus sailed; when the +massacre at Lexington was "news." It is falling now; it will still be +falling when all these things shall have sunk down the afternoon of +history, and the twilight of tradition, and been swallowed up in the +thick night of oblivion. Has everything a purpose and a mission? Did +this drop fall patiently during five thousand years to be ready for +this flitting human insect's need? and has it another important object +to accomplish ten thousand years to come? No matter. It is many and +many a year since the hapless half-breed scooped out the stone to catch +the priceless drops, but to this day the tourist stares longest at that +pathetic stone and that slow-dropping water when he comes to see the +wonders of McDougal's cave. Injun Joe's cup stands first in the list of +the cavern's marvels; even "Aladdin's Palace" cannot rival it. + +Injun Joe was buried near the mouth of the cave; and people flocked +there in boats and wagons from the towns and from all the farms and +hamlets for seven miles around; they brought their children, and all +sorts of provisions, and confessed that they had had almost as +satisfactory a time at the funeral as they could have had at the +hanging. + +This funeral stopped the further growth of one thing--the petition to +the governor for Injun Joe's pardon. The petition had been largely +signed; many tearful and eloquent meetings had been held, and a +committee of sappy women been appointed to go in deep mourning and wail +around the governor, and implore him to be a merciful ass and trample +his duty under foot. Injun Joe was believed to have killed five +citizens of the village, but what of that? If he had been Satan himself +there would have been plenty of weaklings ready to scribble their names +to a pardon-petition, and drip a tear on it from their permanently +impaired and leaky water-works. + +The morning after the funeral Tom took Huck to a private place to have +an important talk. Huck had learned all about Tom's adventure from the +Welshman and the Widow Douglas, by this time, but Tom said he reckoned +there was one thing they had not told him; that thing was what he +wanted to talk about now. Huck's face saddened. He said: + +"I know what it is. You got into No. 2 and never found anything but +whiskey. Nobody told me it was you; but I just knowed it must 'a' ben +you, soon as I heard 'bout that whiskey business; and I knowed you +hadn't got the money becuz you'd 'a' got at me some way or other and +told me even if you was mum to everybody else. Tom, something's always +told me we'd never get holt of that swag." + +"Why, Huck, I never told on that tavern-keeper. YOU know his tavern +was all right the Saturday I went to the picnic. Don't you remember you +was to watch there that night?" + +"Oh yes! Why, it seems 'bout a year ago. It was that very night that I +follered Injun Joe to the widder's." + +"YOU followed him?" + +"Yes--but you keep mum. I reckon Injun Joe's left friends behind him, +and I don't want 'em souring on me and doing me mean tricks. If it +hadn't ben for me he'd be down in Texas now, all right." + +Then Huck told his entire adventure in confidence to Tom, who had only +heard of the Welshman's part of it before. + +"Well," said Huck, presently, coming back to the main question, +"whoever nipped the whiskey in No. 2, nipped the money, too, I reckon +--anyways it's a goner for us, Tom." + +"Huck, that money wasn't ever in No. 2!" + +"What!" Huck searched his comrade's face keenly. "Tom, have you got on +the track of that money again?" + +"Huck, it's in the cave!" + +Huck's eyes blazed. + +"Say it again, Tom." + +"The money's in the cave!" + +"Tom--honest injun, now--is it fun, or earnest?" + +"Earnest, Huck--just as earnest as ever I was in my life. Will you go +in there with me and help get it out?" + +"I bet I will! I will if it's where we can blaze our way to it and not +get lost." + +"Huck, we can do that without the least little bit of trouble in the +world." + +"Good as wheat! What makes you think the money's--" + +"Huck, you just wait till we get in there. If we don't find it I'll +agree to give you my drum and every thing I've got in the world. I +will, by jings." + +"All right--it's a whiz. When do you say?" + +"Right now, if you say it. Are you strong enough?" + +"Is it far in the cave? I ben on my pins a little, three or four days, +now, but I can't walk more'n a mile, Tom--least I don't think I could." + +"It's about five mile into there the way anybody but me would go, +Huck, but there's a mighty short cut that they don't anybody but me +know about. Huck, I'll take you right to it in a skiff. I'll float the +skiff down there, and I'll pull it back again all by myself. You +needn't ever turn your hand over." + +"Less start right off, Tom." + +"All right. We want some bread and meat, and our pipes, and a little +bag or two, and two or three kite-strings, and some of these +new-fangled things they call lucifer matches. I tell you, many's +the time I wished I had some when I was in there before." + +A trifle after noon the boys borrowed a small skiff from a citizen who +was absent, and got under way at once. When they were several miles +below "Cave Hollow," Tom said: + +"Now you see this bluff here looks all alike all the way down from the +cave hollow--no houses, no wood-yards, bushes all alike. But do you see +that white place up yonder where there's been a landslide? Well, that's +one of my marks. We'll get ashore, now." + +They landed. + +"Now, Huck, where we're a-standing you could touch that hole I got out +of with a fishing-pole. See if you can find it." + +Huck searched all the place about, and found nothing. Tom proudly +marched into a thick clump of sumach bushes and said: + +"Here you are! Look at it, Huck; it's the snuggest hole in this +country. You just keep mum about it. All along I've been wanting to be +a robber, but I knew I'd got to have a thing like this, and where to +run across it was the bother. We've got it now, and we'll keep it +quiet, only we'll let Joe Harper and Ben Rogers in--because of course +there's got to be a Gang, or else there wouldn't be any style about it. +Tom Sawyer's Gang--it sounds splendid, don't it, Huck?" + +"Well, it just does, Tom. And who'll we rob?" + +"Oh, most anybody. Waylay people--that's mostly the way." + +"And kill them?" + +"No, not always. Hive them in the cave till they raise a ransom." + +"What's a ransom?" + +"Money. You make them raise all they can, off'n their friends; and +after you've kept them a year, if it ain't raised then you kill them. +That's the general way. Only you don't kill the women. You shut up the +women, but you don't kill them. They're always beautiful and rich, and +awfully scared. You take their watches and things, but you always take +your hat off and talk polite. They ain't anybody as polite as robbers +--you'll see that in any book. Well, the women get to loving you, and +after they've been in the cave a week or two weeks they stop crying and +after that you couldn't get them to leave. If you drove them out they'd +turn right around and come back. It's so in all the books." + +"Why, it's real bully, Tom. I believe it's better'n to be a pirate." + +"Yes, it's better in some ways, because it's close to home and +circuses and all that." + +By this time everything was ready and the boys entered the hole, Tom +in the lead. They toiled their way to the farther end of the tunnel, +then made their spliced kite-strings fast and moved on. A few steps +brought them to the spring, and Tom felt a shudder quiver all through +him. He showed Huck the fragment of candle-wick perched on a lump of +clay against the wall, and described how he and Becky had watched the +flame struggle and expire. + +The boys began to quiet down to whispers, now, for the stillness and +gloom of the place oppressed their spirits. They went on, and presently +entered and followed Tom's other corridor until they reached the +"jumping-off place." The candles revealed the fact that it was not +really a precipice, but only a steep clay hill twenty or thirty feet +high. Tom whispered: + +"Now I'll show you something, Huck." + +He held his candle aloft and said: + +"Look as far around the corner as you can. Do you see that? There--on +the big rock over yonder--done with candle-smoke." + +"Tom, it's a CROSS!" + +"NOW where's your Number Two? 'UNDER THE CROSS,' hey? Right yonder's +where I saw Injun Joe poke up his candle, Huck!" + +Huck stared at the mystic sign awhile, and then said with a shaky voice: + +"Tom, less git out of here!" + +"What! and leave the treasure?" + +"Yes--leave it. Injun Joe's ghost is round about there, certain." + +"No it ain't, Huck, no it ain't. It would ha'nt the place where he +died--away out at the mouth of the cave--five mile from here." + +"No, Tom, it wouldn't. It would hang round the money. I know the ways +of ghosts, and so do you." + +Tom began to fear that Huck was right. Misgivings gathered in his +mind. But presently an idea occurred to him-- + +"Lookyhere, Huck, what fools we're making of ourselves! Injun Joe's +ghost ain't a going to come around where there's a cross!" + +The point was well taken. It had its effect. + +"Tom, I didn't think of that. But that's so. It's luck for us, that +cross is. I reckon we'll climb down there and have a hunt for that box." + +Tom went first, cutting rude steps in the clay hill as he descended. +Huck followed. Four avenues opened out of the small cavern which the +great rock stood in. The boys examined three of them with no result. +They found a small recess in the one nearest the base of the rock, with +a pallet of blankets spread down in it; also an old suspender, some +bacon rind, and the well-gnawed bones of two or three fowls. But there +was no money-box. The lads searched and researched this place, but in +vain. Tom said: + +"He said UNDER the cross. Well, this comes nearest to being under the +cross. It can't be under the rock itself, because that sets solid on +the ground." + +They searched everywhere once more, and then sat down discouraged. +Huck could suggest nothing. By-and-by Tom said: + +"Lookyhere, Huck, there's footprints and some candle-grease on the +clay about one side of this rock, but not on the other sides. Now, +what's that for? I bet you the money IS under the rock. I'm going to +dig in the clay." + +"That ain't no bad notion, Tom!" said Huck with animation. + +Tom's "real Barlow" was out at once, and he had not dug four inches +before he struck wood. + +"Hey, Huck!--you hear that?" + +Huck began to dig and scratch now. Some boards were soon uncovered and +removed. They had concealed a natural chasm which led under the rock. +Tom got into this and held his candle as far under the rock as he +could, but said he could not see to the end of the rift. He proposed to +explore. He stooped and passed under; the narrow way descended +gradually. He followed its winding course, first to the right, then to +the left, Huck at his heels. Tom turned a short curve, by-and-by, and +exclaimed: + +"My goodness, Huck, lookyhere!" + +It was the treasure-box, sure enough, occupying a snug little cavern, +along with an empty powder-keg, a couple of guns in leather cases, two +or three pairs of old moccasins, a leather belt, and some other rubbish +well soaked with the water-drip. + +"Got it at last!" said Huck, ploughing among the tarnished coins with +his hand. "My, but we're rich, Tom!" + +"Huck, I always reckoned we'd get it. It's just too good to believe, +but we HAVE got it, sure! Say--let's not fool around here. Let's snake +it out. Lemme see if I can lift the box." + +It weighed about fifty pounds. Tom could lift it, after an awkward +fashion, but could not carry it conveniently. + +"I thought so," he said; "THEY carried it like it was heavy, that day +at the ha'nted house. I noticed that. I reckon I was right to think of +fetching the little bags along." + +The money was soon in the bags and the boys took it up to the cross +rock. + +"Now less fetch the guns and things," said Huck. + +"No, Huck--leave them there. They're just the tricks to have when we +go to robbing. We'll keep them there all the time, and we'll hold our +orgies there, too. It's an awful snug place for orgies." + +"What orgies?" + +"I dono. But robbers always have orgies, and of course we've got to +have them, too. Come along, Huck, we've been in here a long time. It's +getting late, I reckon. I'm hungry, too. We'll eat and smoke when we +get to the skiff." + +They presently emerged into the clump of sumach bushes, looked warily +out, found the coast clear, and were soon lunching and smoking in the +skiff. As the sun dipped toward the horizon they pushed out and got +under way. Tom skimmed up the shore through the long twilight, chatting +cheerily with Huck, and landed shortly after dark. + +"Now, Huck," said Tom, "we'll hide the money in the loft of the +widow's woodshed, and I'll come up in the morning and we'll count it +and divide, and then we'll hunt up a place out in the woods for it +where it will be safe. Just you lay quiet here and watch the stuff till +I run and hook Benny Taylor's little wagon; I won't be gone a minute." + +He disappeared, and presently returned with the wagon, put the two +small sacks into it, threw some old rags on top of them, and started +off, dragging his cargo behind him. When the boys reached the +Welshman's house, they stopped to rest. Just as they were about to move +on, the Welshman stepped out and said: + +"Hallo, who's that?" + +"Huck and Tom Sawyer." + +"Good! Come along with me, boys, you are keeping everybody waiting. +Here--hurry up, trot ahead--I'll haul the wagon for you. Why, it's not +as light as it might be. Got bricks in it?--or old metal?" + +"Old metal," said Tom. + +"I judged so; the boys in this town will take more trouble and fool +away more time hunting up six bits' worth of old iron to sell to the +foundry than they would to make twice the money at regular work. But +that's human nature--hurry along, hurry along!" + +The boys wanted to know what the hurry was about. + +"Never mind; you'll see, when we get to the Widow Douglas'." + +Huck said with some apprehension--for he was long used to being +falsely accused: + +"Mr. Jones, we haven't been doing nothing." + +The Welshman laughed. + +"Well, I don't know, Huck, my boy. I don't know about that. Ain't you +and the widow good friends?" + +"Yes. Well, she's ben good friends to me, anyway." + +"All right, then. What do you want to be afraid for?" + +This question was not entirely answered in Huck's slow mind before he +found himself pushed, along with Tom, into Mrs. Douglas' drawing-room. +Mr. Jones left the wagon near the door and followed. + +The place was grandly lighted, and everybody that was of any +consequence in the village was there. The Thatchers were there, the +Harpers, the Rogerses, Aunt Polly, Sid, Mary, the minister, the editor, +and a great many more, and all dressed in their best. The widow +received the boys as heartily as any one could well receive two such +looking beings. They were covered with clay and candle-grease. Aunt +Polly blushed crimson with humiliation, and frowned and shook her head +at Tom. Nobody suffered half as much as the two boys did, however. Mr. +Jones said: + +"Tom wasn't at home, yet, so I gave him up; but I stumbled on him and +Huck right at my door, and so I just brought them along in a hurry." + +"And you did just right," said the widow. "Come with me, boys." + +She took them to a bedchamber and said: + +"Now wash and dress yourselves. Here are two new suits of clothes +--shirts, socks, everything complete. They're Huck's--no, no thanks, +Huck--Mr. Jones bought one and I the other. But they'll fit both of you. +Get into them. We'll wait--come down when you are slicked up enough." + +Then she left. + + + +CHAPTER XXXIV + +HUCK said: "Tom, we can slope, if we can find a rope. The window ain't +high from the ground." + +"Shucks! what do you want to slope for?" + +"Well, I ain't used to that kind of a crowd. I can't stand it. I ain't +going down there, Tom." + +"Oh, bother! It ain't anything. I don't mind it a bit. I'll take care +of you." + +Sid appeared. + +"Tom," said he, "auntie has been waiting for you all the afternoon. +Mary got your Sunday clothes ready, and everybody's been fretting about +you. Say--ain't this grease and clay, on your clothes?" + +"Now, Mr. Siddy, you jist 'tend to your own business. What's all this +blow-out about, anyway?" + +"It's one of the widow's parties that she's always having. This time +it's for the Welshman and his sons, on account of that scrape they +helped her out of the other night. And say--I can tell you something, +if you want to know." + +"Well, what?" + +"Why, old Mr. Jones is going to try to spring something on the people +here to-night, but I overheard him tell auntie to-day about it, as a +secret, but I reckon it's not much of a secret now. Everybody knows +--the widow, too, for all she tries to let on she don't. Mr. Jones was +bound Huck should be here--couldn't get along with his grand secret +without Huck, you know!" + +"Secret about what, Sid?" + +"About Huck tracking the robbers to the widow's. I reckon Mr. Jones +was going to make a grand time over his surprise, but I bet you it will +drop pretty flat." + +Sid chuckled in a very contented and satisfied way. + +"Sid, was it you that told?" + +"Oh, never mind who it was. SOMEBODY told--that's enough." + +"Sid, there's only one person in this town mean enough to do that, and +that's you. If you had been in Huck's place you'd 'a' sneaked down the +hill and never told anybody on the robbers. You can't do any but mean +things, and you can't bear to see anybody praised for doing good ones. +There--no thanks, as the widow says"--and Tom cuffed Sid's ears and +helped him to the door with several kicks. "Now go and tell auntie if +you dare--and to-morrow you'll catch it!" + +Some minutes later the widow's guests were at the supper-table, and a +dozen children were propped up at little side-tables in the same room, +after the fashion of that country and that day. At the proper time Mr. +Jones made his little speech, in which he thanked the widow for the +honor she was doing himself and his sons, but said that there was +another person whose modesty-- + +And so forth and so on. He sprung his secret about Huck's share in the +adventure in the finest dramatic manner he was master of, but the +surprise it occasioned was largely counterfeit and not as clamorous and +effusive as it might have been under happier circumstances. However, +the widow made a pretty fair show of astonishment, and heaped so many +compliments and so much gratitude upon Huck that he almost forgot the +nearly intolerable discomfort of his new clothes in the entirely +intolerable discomfort of being set up as a target for everybody's gaze +and everybody's laudations. + +The widow said she meant to give Huck a home under her roof and have +him educated; and that when she could spare the money she would start +him in business in a modest way. Tom's chance was come. He said: + +"Huck don't need it. Huck's rich." + +Nothing but a heavy strain upon the good manners of the company kept +back the due and proper complimentary laugh at this pleasant joke. But +the silence was a little awkward. Tom broke it: + +"Huck's got money. Maybe you don't believe it, but he's got lots of +it. Oh, you needn't smile--I reckon I can show you. You just wait a +minute." + +Tom ran out of doors. The company looked at each other with a +perplexed interest--and inquiringly at Huck, who was tongue-tied. + +"Sid, what ails Tom?" said Aunt Polly. "He--well, there ain't ever any +making of that boy out. I never--" + +Tom entered, struggling with the weight of his sacks, and Aunt Polly +did not finish her sentence. Tom poured the mass of yellow coin upon +the table and said: + +"There--what did I tell you? Half of it's Huck's and half of it's mine!" + +The spectacle took the general breath away. All gazed, nobody spoke +for a moment. Then there was a unanimous call for an explanation. Tom +said he could furnish it, and he did. The tale was long, but brimful of +interest. There was scarcely an interruption from any one to break the +charm of its flow. When he had finished, Mr. Jones said: + +"I thought I had fixed up a little surprise for this occasion, but it +don't amount to anything now. This one makes it sing mighty small, I'm +willing to allow." + +The money was counted. The sum amounted to a little over twelve +thousand dollars. It was more than any one present had ever seen at one +time before, though several persons were there who were worth +considerably more than that in property. + + + +CHAPTER XXXV + +THE reader may rest satisfied that Tom's and Huck's windfall made a +mighty stir in the poor little village of St. Petersburg. So vast a +sum, all in actual cash, seemed next to incredible. It was talked +about, gloated over, glorified, until the reason of many of the +citizens tottered under the strain of the unhealthy excitement. Every +"haunted" house in St. Petersburg and the neighboring villages was +dissected, plank by plank, and its foundations dug up and ransacked for +hidden treasure--and not by boys, but men--pretty grave, unromantic +men, too, some of them. Wherever Tom and Huck appeared they were +courted, admired, stared at. The boys were not able to remember that +their remarks had possessed weight before; but now their sayings were +treasured and repeated; everything they did seemed somehow to be +regarded as remarkable; they had evidently lost the power of doing and +saying commonplace things; moreover, their past history was raked up +and discovered to bear marks of conspicuous originality. The village +paper published biographical sketches of the boys. + +The Widow Douglas put Huck's money out at six per cent., and Judge +Thatcher did the same with Tom's at Aunt Polly's request. Each lad had +an income, now, that was simply prodigious--a dollar for every week-day +in the year and half of the Sundays. It was just what the minister got +--no, it was what he was promised--he generally couldn't collect it. A +dollar and a quarter a week would board, lodge, and school a boy in +those old simple days--and clothe him and wash him, too, for that +matter. + +Judge Thatcher had conceived a great opinion of Tom. He said that no +commonplace boy would ever have got his daughter out of the cave. When +Becky told her father, in strict confidence, how Tom had taken her +whipping at school, the Judge was visibly moved; and when she pleaded +grace for the mighty lie which Tom had told in order to shift that +whipping from her shoulders to his own, the Judge said with a fine +outburst that it was a noble, a generous, a magnanimous lie--a lie that +was worthy to hold up its head and march down through history breast to +breast with George Washington's lauded Truth about the hatchet! Becky +thought her father had never looked so tall and so superb as when he +walked the floor and stamped his foot and said that. She went straight +off and told Tom about it. + +Judge Thatcher hoped to see Tom a great lawyer or a great soldier some +day. He said he meant to look to it that Tom should be admitted to the +National Military Academy and afterward trained in the best law school +in the country, in order that he might be ready for either career or +both. + +Huck Finn's wealth and the fact that he was now under the Widow +Douglas' protection introduced him into society--no, dragged him into +it, hurled him into it--and his sufferings were almost more than he +could bear. The widow's servants kept him clean and neat, combed and +brushed, and they bedded him nightly in unsympathetic sheets that had +not one little spot or stain which he could press to his heart and know +for a friend. He had to eat with a knife and fork; he had to use +napkin, cup, and plate; he had to learn his book, he had to go to +church; he had to talk so properly that speech was become insipid in +his mouth; whithersoever he turned, the bars and shackles of +civilization shut him in and bound him hand and foot. + +He bravely bore his miseries three weeks, and then one day turned up +missing. For forty-eight hours the widow hunted for him everywhere in +great distress. The public were profoundly concerned; they searched +high and low, they dragged the river for his body. Early the third +morning Tom Sawyer wisely went poking among some old empty hogsheads +down behind the abandoned slaughter-house, and in one of them he found +the refugee. Huck had slept there; he had just breakfasted upon some +stolen odds and ends of food, and was lying off, now, in comfort, with +his pipe. He was unkempt, uncombed, and clad in the same old ruin of +rags that had made him picturesque in the days when he was free and +happy. Tom routed him out, told him the trouble he had been causing, +and urged him to go home. Huck's face lost its tranquil content, and +took a melancholy cast. He said: + +"Don't talk about it, Tom. I've tried it, and it don't work; it don't +work, Tom. It ain't for me; I ain't used to it. The widder's good to +me, and friendly; but I can't stand them ways. She makes me get up just +at the same time every morning; she makes me wash, they comb me all to +thunder; she won't let me sleep in the woodshed; I got to wear them +blamed clothes that just smothers me, Tom; they don't seem to any air +git through 'em, somehow; and they're so rotten nice that I can't set +down, nor lay down, nor roll around anywher's; I hain't slid on a +cellar-door for--well, it 'pears to be years; I got to go to church and +sweat and sweat--I hate them ornery sermons! I can't ketch a fly in +there, I can't chaw. I got to wear shoes all Sunday. The widder eats by +a bell; she goes to bed by a bell; she gits up by a bell--everything's +so awful reg'lar a body can't stand it." + +"Well, everybody does that way, Huck." + +"Tom, it don't make no difference. I ain't everybody, and I can't +STAND it. It's awful to be tied up so. And grub comes too easy--I don't +take no interest in vittles, that way. I got to ask to go a-fishing; I +got to ask to go in a-swimming--dern'd if I hain't got to ask to do +everything. Well, I'd got to talk so nice it wasn't no comfort--I'd got +to go up in the attic and rip out awhile, every day, to git a taste in +my mouth, or I'd a died, Tom. The widder wouldn't let me smoke; she +wouldn't let me yell, she wouldn't let me gape, nor stretch, nor +scratch, before folks--" [Then with a spasm of special irritation and +injury]--"And dad fetch it, she prayed all the time! I never see such a +woman! I HAD to shove, Tom--I just had to. And besides, that school's +going to open, and I'd a had to go to it--well, I wouldn't stand THAT, +Tom. Looky here, Tom, being rich ain't what it's cracked up to be. It's +just worry and worry, and sweat and sweat, and a-wishing you was dead +all the time. Now these clothes suits me, and this bar'l suits me, and +I ain't ever going to shake 'em any more. Tom, I wouldn't ever got into +all this trouble if it hadn't 'a' ben for that money; now you just take +my sheer of it along with your'n, and gimme a ten-center sometimes--not +many times, becuz I don't give a dern for a thing 'thout it's tollable +hard to git--and you go and beg off for me with the widder." + +"Oh, Huck, you know I can't do that. 'Tain't fair; and besides if +you'll try this thing just a while longer you'll come to like it." + +"Like it! Yes--the way I'd like a hot stove if I was to set on it long +enough. No, Tom, I won't be rich, and I won't live in them cussed +smothery houses. I like the woods, and the river, and hogsheads, and +I'll stick to 'em, too. Blame it all! just as we'd got guns, and a +cave, and all just fixed to rob, here this dern foolishness has got to +come up and spile it all!" + +Tom saw his opportunity-- + +"Lookyhere, Huck, being rich ain't going to keep me back from turning +robber." + +"No! Oh, good-licks; are you in real dead-wood earnest, Tom?" + +"Just as dead earnest as I'm sitting here. But Huck, we can't let you +into the gang if you ain't respectable, you know." + +Huck's joy was quenched. + +"Can't let me in, Tom? Didn't you let me go for a pirate?" + +"Yes, but that's different. A robber is more high-toned than what a +pirate is--as a general thing. In most countries they're awful high up +in the nobility--dukes and such." + +"Now, Tom, hain't you always ben friendly to me? You wouldn't shet me +out, would you, Tom? You wouldn't do that, now, WOULD you, Tom?" + +"Huck, I wouldn't want to, and I DON'T want to--but what would people +say? Why, they'd say, 'Mph! Tom Sawyer's Gang! pretty low characters in +it!' They'd mean you, Huck. You wouldn't like that, and I wouldn't." + +Huck was silent for some time, engaged in a mental struggle. Finally +he said: + +"Well, I'll go back to the widder for a month and tackle it and see if +I can come to stand it, if you'll let me b'long to the gang, Tom." + +"All right, Huck, it's a whiz! Come along, old chap, and I'll ask the +widow to let up on you a little, Huck." + +"Will you, Tom--now will you? That's good. If she'll let up on some of +the roughest things, I'll smoke private and cuss private, and crowd +through or bust. When you going to start the gang and turn robbers?" + +"Oh, right off. We'll get the boys together and have the initiation +to-night, maybe." + +"Have the which?" + +"Have the initiation." + +"What's that?" + +"It's to swear to stand by one another, and never tell the gang's +secrets, even if you're chopped all to flinders, and kill anybody and +all his family that hurts one of the gang." + +"That's gay--that's mighty gay, Tom, I tell you." + +"Well, I bet it is. And all that swearing's got to be done at +midnight, in the lonesomest, awfulest place you can find--a ha'nted +house is the best, but they're all ripped up now." + +"Well, midnight's good, anyway, Tom." + +"Yes, so it is. And you've got to swear on a coffin, and sign it with +blood." + +"Now, that's something LIKE! Why, it's a million times bullier than +pirating. I'll stick to the widder till I rot, Tom; and if I git to be +a reg'lar ripper of a robber, and everybody talking 'bout it, I reckon +she'll be proud she snaked me in out of the wet." + + + +CONCLUSION + +SO endeth this chronicle. It being strictly a history of a BOY, it +must stop here; the story could not go much further without becoming +the history of a MAN. When one writes a novel about grown people, he +knows exactly where to stop--that is, with a marriage; but when he +writes of juveniles, he must stop where he best can. + +Most of the characters that perform in this book still live, and are +prosperous and happy. Some day it may seem worth while to take up the +story of the younger ones again and see what sort of men and women they +turned out to be; therefore it will be wisest not to reveal any of that +part of their lives at present. diff --git a/libgo/go/net/textproto/header.go b/libgo/go/net/textproto/header.go index 7fb32f80..2e2752a 100644 --- a/libgo/go/net/textproto/header.go +++ b/libgo/go/net/textproto/header.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -16,7 +16,7 @@ func (h MIMEHeader) Add(key, value string) { } // Set sets the header entries associated with key to -// the single element value. It replaces any existing +// the single element value. It replaces any existing // values associated with key. func (h MIMEHeader) Set(key, value string) { h[CanonicalMIMEHeaderKey(key)] = []string{value} @@ -24,7 +24,7 @@ func (h MIMEHeader) Set(key, value string) { // Get gets the first value associated with the given key. // If there are no values associated with the key, Get returns "". -// Get is a convenience method. For more complex queries, +// Get is a convenience method. For more complex queries, // access the map directly. func (h MIMEHeader) Get(key string) string { if h == nil { diff --git a/libgo/go/net/textproto/pipeline.go b/libgo/go/net/textproto/pipeline.go index ca50edd..2e28321 100644 --- a/libgo/go/net/textproto/pipeline.go +++ b/libgo/go/net/textproto/pipeline.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -66,8 +66,8 @@ func (p *Pipeline) EndResponse(id uint) { } // A sequencer schedules a sequence of numbered events that must -// happen in order, one after the other. The event numbering must start -// at 0 and increment without skipping. The event number wraps around +// happen in order, one after the other. The event numbering must start +// at 0 and increment without skipping. The event number wraps around // safely as long as there are not 2^32 simultaneous events pending. type sequencer struct { mu sync.Mutex diff --git a/libgo/go/net/textproto/reader.go b/libgo/go/net/textproto/reader.go index 91bbb57..e07d1d6 100644 --- a/libgo/go/net/textproto/reader.go +++ b/libgo/go/net/textproto/reader.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -71,7 +71,7 @@ func (r *Reader) readLineSlice() ([]byte, error) { // ReadContinuedLine reads a possibly continued line from r, // eliding the final trailing ASCII white space. // Lines after the first are considered continuations if they -// begin with a space or tab character. In the returned data, +// begin with a space or tab character. In the returned data, // continuation lines are separated from the previous line // only by a single space: the newline and leading white space // are removed. @@ -204,7 +204,7 @@ func parseCodeLine(line string, expectCode int) (code int, continued bool, messa // ReadCodeLine reads a response code line of the form // code message // where code is a three-digit status code and the message -// extends to the rest of the line. An example of such a line is: +// extends to the rest of the line. An example of such a line is: // 220 plan9.bell-labs.com ESMTP // // If the prefix of the status does not match the digits in expectCode, @@ -366,7 +366,7 @@ func (d *dotReader) Read(b []byte) (n int, err error) { d.state = stateBeginLine break } - // Not part of \r\n. Emit saved \r + // Not part of \r\n. Emit saved \r br.UnreadByte() c = '\r' d.state = stateData @@ -552,9 +552,9 @@ func (r *Reader) upcomingHeaderNewlines() (n int) { } // CanonicalMIMEHeaderKey returns the canonical format of the -// MIME header key s. The canonicalization converts the first +// MIME header key s. The canonicalization converts the first // letter and any letter following a hyphen to upper case; -// the rest are converted to lowercase. For example, the +// the rest are converted to lowercase. For example, the // canonical key for "accept-encoding" is "Accept-Encoding". // MIME header keys are assumed to be ASCII only. // If s contains a space or invalid header field bytes, it is @@ -581,18 +581,14 @@ func CanonicalMIMEHeaderKey(s string) string { const toLower = 'a' - 'A' // validHeaderFieldByte reports whether b is a valid byte in a header -// field key. This is actually stricter than RFC 7230, which says: +// field name. RFC 7230 says: +// header-field = field-name ":" OWS field-value OWS +// field-name = token // tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / // "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA // token = 1*tchar -// TODO: revisit in Go 1.6+ and possibly expand this. But note that many -// servers have historically dropped '_' to prevent ambiguities when mapping -// to CGI environment variables. func validHeaderFieldByte(b byte) bool { - return ('A' <= b && b <= 'Z') || - ('a' <= b && b <= 'z') || - ('0' <= b && b <= '9') || - b == '-' + return int(b) < len(isTokenTable) && isTokenTable[b] } // canonicalMIMEHeaderKey is like CanonicalMIMEHeaderKey but is @@ -682,3 +678,85 @@ func init() { commonHeader[v] = v } } + +// isTokenTable is a copy of net/http/lex.go's isTokenTable. +// See https://httpwg.github.io/specs/rfc7230.html#rule.token.separators +var isTokenTable = [127]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, +} diff --git a/libgo/go/net/textproto/reader_test.go b/libgo/go/net/textproto/reader_test.go index 93d7939..0c53d48 100644 --- a/libgo/go/net/textproto/reader_test.go +++ b/libgo/go/net/textproto/reader_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -25,6 +25,12 @@ var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{ {"user-agent", "User-Agent"}, {"USER-AGENT", "User-Agent"}, + // Other valid tchar bytes in tokens: + {"foo-bar_baz", "Foo-Bar_baz"}, + {"foo-bar$baz", "Foo-Bar$baz"}, + {"foo-bar~baz", "Foo-Bar~baz"}, + {"foo-bar*baz", "Foo-Bar*baz"}, + // Non-ASCII or anything with spaces or non-token chars is unchanged: {"üser-agenT", "üser-agenT"}, {"a B", "a B"}, diff --git a/libgo/go/net/textproto/textproto.go b/libgo/go/net/textproto/textproto.go index 026eb02..8fd781e 100644 --- a/libgo/go/net/textproto/textproto.go +++ b/libgo/go/net/textproto/textproto.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -87,7 +87,7 @@ func Dial(network, addr string) (*Conn, error) { } // Cmd is a convenience method that sends a command after -// waiting its turn in the pipeline. The command text is the +// waiting its turn in the pipeline. The command text is the // result of formatting format with args and appending \r\n. // Cmd returns the id of the command, for use with StartResponse and EndResponse. // diff --git a/libgo/go/net/textproto/writer.go b/libgo/go/net/textproto/writer.go index 03e2fd6..1bc5974 100644 --- a/libgo/go/net/textproto/writer.go +++ b/libgo/go/net/textproto/writer.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -36,7 +36,7 @@ func (w *Writer) PrintfLine(format string, args ...interface{}) error { // DotWriter returns a writer that can be used to write a dot-encoding to w. // It takes care of inserting leading dots when necessary, // translating line-ending \n into \r\n, and adding the final .\r\n line -// when the DotWriter is closed. The caller should close the +// when the DotWriter is closed. The caller should close the // DotWriter before the next call to a method on w. // // See the documentation for Reader's DotReader method for details about dot-encoding. diff --git a/libgo/go/net/textproto/writer_test.go b/libgo/go/net/textproto/writer_test.go index e03ab5e..ac03669 100644 --- a/libgo/go/net/textproto/writer_test.go +++ b/libgo/go/net/textproto/writer_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/timeout_test.go b/libgo/go/net/timeout_test.go index 98e3164..ed26f2a 100644 --- a/libgo/go/net/timeout_test.go +++ b/libgo/go/net/timeout_test.go @@ -5,7 +5,9 @@ package net import ( + "context" "fmt" + "internal/testenv" "io" "io/ioutil" "net/internal/socktest" @@ -26,6 +28,8 @@ var dialTimeoutTests = []struct { {-5 * time.Second, 0, -5 * time.Second, 100 * time.Millisecond}, {0, -5 * time.Second, -5 * time.Second, 100 * time.Millisecond}, {-5 * time.Second, 5 * time.Second, -5 * time.Second, 100 * time.Millisecond}, // timeout over deadline + {-1 << 63, 0, time.Second, 100 * time.Millisecond}, + {0, -1 << 63, time.Second, 100 * time.Millisecond}, {50 * time.Millisecond, 0, 100 * time.Millisecond, time.Second}, {0, 50 * time.Millisecond, 100 * time.Millisecond, time.Second}, @@ -38,19 +42,6 @@ func TestDialTimeout(t *testing.T) { defer func() { testHookDialChannel = origTestHookDialChannel }() defer sw.Set(socktest.FilterConnect, nil) - // Avoid tracking open-close jitterbugs between netFD and - // socket that leads to confusion of information inside - // socktest.Switch. - // It may happen when the Dial call bumps against TCP - // simultaneous open. See selfConnect in tcpsock_posix.go. - defer func() { - sw.Set(socktest.FilterClose, nil) - forceCloseSockets() - }() - sw.Set(socktest.FilterClose, func(so *socktest.Status) (socktest.AfterFilter, error) { - return nil, errTimedout - }) - for i, tt := range dialTimeoutTests { switch runtime.GOOS { case "plan9", "windows": @@ -99,6 +90,56 @@ func TestDialTimeout(t *testing.T) { } } +var dialTimeoutMaxDurationTests = []struct { + timeout time.Duration + delta time.Duration // for deadline +}{ + // Large timeouts that will overflow an int64 unix nanos. + {1<<63 - 1, 0}, + {0, 1<<63 - 1}, +} + +func TestDialTimeoutMaxDuration(t *testing.T) { + if runtime.GOOS == "openbsd" { + testenv.SkipFlaky(t, 15157) + } + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + + for i, tt := range dialTimeoutMaxDurationTests { + ch := make(chan error) + max := time.NewTimer(250 * time.Millisecond) + defer max.Stop() + go func() { + d := Dialer{Timeout: tt.timeout} + if tt.delta != 0 { + d.Deadline = time.Now().Add(tt.delta) + } + c, err := d.Dial(ln.Addr().Network(), ln.Addr().String()) + if err == nil { + c.Close() + } + ch <- err + }() + + select { + case <-max.C: + t.Fatalf("#%d: Dial didn't return in an expected time", i) + case err := <-ch: + if perr := parseDialError(err); perr != nil { + t.Error(perr) + } + if err != nil { + t.Errorf("#%d: %v", i, err) + } + } + } +} + var acceptTimeoutTests = []struct { timeout time.Duration xerrs [2]error // expected errors in transition @@ -124,10 +165,13 @@ func TestAcceptTimeout(t *testing.T) { } defer ln.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() for i, tt := range acceptTimeoutTests { if tt.timeout < 0 { go func() { - c, err := Dial(ln.Addr().Network(), ln.Addr().String()) + var d Dialer + c, err := d.DialContext(ctx, ln.Addr().Network(), ln.Addr().String()) if err != nil { t.Error(err) return @@ -261,8 +305,6 @@ var readTimeoutTests = []struct { } func TestReadTimeout(t *testing.T) { - t.Parallel() - switch runtime.GOOS { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) diff --git a/libgo/go/net/udpsock.go b/libgo/go/net/udpsock.go index 9292133..980f67c 100644 --- a/libgo/go/net/udpsock.go +++ b/libgo/go/net/udpsock.go @@ -1,9 +1,14 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import ( + "context" + "syscall" +) + // UDPAddr represents the address of a UDP end point. type UDPAddr struct { IP IP @@ -53,9 +58,185 @@ func ResolveUDPAddr(net, addr string) (*UDPAddr, error) { default: return nil, UnknownNetworkError(net) } - addrs, err := internetAddrList(net, addr, noDeadline) + addrs, err := internetAddrList(context.Background(), net, addr) if err != nil { return nil, err } return addrs.first(isIPv4).(*UDPAddr), nil } + +// UDPConn is the implementation of the Conn and PacketConn interfaces +// for UDP network connections. +type UDPConn struct { + conn +} + +// ReadFromUDP reads a UDP packet from c, copying the payload into b. +// It returns the number of bytes copied into b and the return address +// that was on the packet. +// +// ReadFromUDP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *UDPConn) ReadFromUDP(b []byte) (int, *UDPAddr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, addr, err +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + if addr == nil { + return n, nil, err + } + return n, addr, err +} + +// ReadMsgUDP reads a packet from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number +// of bytes copied into b, the number of bytes copied into oob, the +// flags that were set on the packet and the source address of the +// packet. +func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { + if !c.ok() { + return 0, 0, 0, nil, syscall.EINVAL + } + n, oobn, flags, addr, err = c.readMsg(b, oob) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return +} + +// WriteToUDP writes a UDP packet to addr via c, copying the payload +// from b. +// +// WriteToUDP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.writeTo(b, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return n, err +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*UDPAddr) + if !ok { + return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} + } + n, err := c.writeTo(b, a) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: a.opAddr(), Err: err} + } + return n, err +} + +// WriteMsgUDP writes a packet to addr via c if c isn't connected, or +// to c's remote destination address if c is connected (in which case +// addr must be nil). The payload is copied from b and the associated +// out-of-band data is copied from oob. It returns the number of +// payload and out-of-band bytes written. +func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { + if !c.ok() { + return 0, 0, syscall.EINVAL + } + n, oobn, err = c.writeMsg(b, oob, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return +} + +func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } + +// DialUDP connects to the remote address raddr on the network net, +// which must be "udp", "udp4", or "udp6". If laddr is not nil, it is +// used as the local address for the connection. +func DialUDP(net string, laddr, raddr *UDPAddr) (*UDPConn, error) { + switch net { + case "udp", "udp4", "udp6": + default: + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + } + if raddr == nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + } + c, err := dialUDP(context.Background(), net, laddr, raddr) + if err != nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + } + return c, nil +} + +// ListenUDP listens for incoming UDP packets addressed to the local +// address laddr. Net must be "udp", "udp4", or "udp6". If laddr has +// a port of 0, ListenUDP will choose an available port. +// The LocalAddr method of the returned UDPConn can be used to +// discover the port. The returned connection's ReadFrom and WriteTo +// methods can be used to receive and send UDP packets with per-packet +// addressing. +func ListenUDP(net string, laddr *UDPAddr) (*UDPConn, error) { + switch net { + case "udp", "udp4", "udp6": + default: + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + } + if laddr == nil { + laddr = &UDPAddr{} + } + c, err := listenUDP(context.Background(), net, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return c, nil +} + +// ListenMulticastUDP listens for incoming multicast UDP packets +// addressed to the group address gaddr on the interface ifi. +// Network must be "udp", "udp4" or "udp6". +// ListenMulticastUDP uses the system-assigned multicast interface +// when ifi is nil, although this is not recommended because the +// assignment depends on platforms and sometimes it might require +// routing configuration. +// +// ListenMulticastUDP is just for convenience of simple, small +// applications. There are golang.org/x/net/ipv4 and +// golang.org/x/net/ipv6 packages for general purpose uses. +func ListenMulticastUDP(network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + switch network { + case "udp", "udp4", "udp6": + default: + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: UnknownNetworkError(network)} + } + if gaddr == nil || gaddr.IP == nil { + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: errMissingAddress} + } + c, err := listenMulticastUDP(context.Background(), network, ifi, gaddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: err} + } + return c, nil +} diff --git a/libgo/go/net/udpsock_plan9.go b/libgo/go/net/udpsock_plan9.go index 1ba57a2..666f206 100644 --- a/libgo/go/net/udpsock_plan9.go +++ b/libgo/go/net/udpsock_plan9.go @@ -1,42 +1,24 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "errors" "os" "syscall" - "time" ) -// UDPConn is the implementation of the Conn and PacketConn interfaces -// for UDP network connections. -type UDPConn struct { - conn -} - -func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } - -// ReadFromUDP reads a UDP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err error) { - if !c.ok() || c.fd.data == nil { - return 0, nil, syscall.EINVAL - } +func (c *UDPConn) readFrom(b []byte) (n int, addr *UDPAddr, err error) { buf := make([]byte, udpHeaderSize+len(b)) - m, err := c.fd.data.Read(buf) + m, err := c.fd.Read(buf) if err != nil { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + return 0, nil, err } if m < udpHeaderSize { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: errors.New("short read reading UDP header")} + return 0, nil, errors.New("short read reading UDP header") } buf = buf[:m] @@ -45,36 +27,13 @@ func (c *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err error) { return n, &UDPAddr{IP: h.raddr, Port: int(h.rport)}, nil } -// ReadFrom implements the PacketConn ReadFrom method. -func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } - return c.ReadFromUDP(b) +func (c *UDPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { + return 0, 0, 0, nil, syscall.EPLAN9 } -// ReadMsgUDP reads a packet from c, copying the payload into b and -// the associated out-of-band data into oob. It returns the number -// of bytes copied into b, the number of bytes copied into oob, the -// flags that were set on the packet and the source address of the -// packet. -func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { - return 0, 0, 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} -} - -// WriteToUDP writes a UDP packet to addr via c, copying the payload -// from b. -// -// WriteToUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { - if !c.ok() || c.fd.data == nil { - return 0, syscall.EINVAL - } +func (c *UDPConn) writeTo(b []byte, addr *UDPAddr) (int, error) { if addr == nil { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} + return 0, errMissingAddress } h := new(udpHeader) h.raddr = addr.IP.To16() @@ -86,53 +45,18 @@ func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { buf := make([]byte, udpHeaderSize+len(b)) i := copy(buf, h.Bytes()) copy(buf[i:], b) - if _, err := c.fd.data.Write(buf); err != nil { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + if _, err := c.fd.Write(buf); err != nil { + return 0, err } return len(b), nil } -// WriteTo implements the PacketConn WriteTo method. -func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } - a, ok := addr.(*UDPAddr) - if !ok { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} - } - return c.WriteToUDP(b, a) +func (c *UDPConn) writeMsg(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { + return 0, 0, syscall.EPLAN9 } -// WriteMsgUDP writes a packet to addr via c if c isn't connected, or -// to c's remote destination address if c is connected (in which case -// addr must be nil). The payload is copied from b and the associated -// out-of-band data is copied from oob. It returns the number of -// payload and out-of-band bytes written. -func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { - return 0, 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} -} - -// DialUDP connects to the remote address raddr on the network net, -// which must be "udp", "udp4", or "udp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialUDP(net string, laddr, raddr *UDPAddr) (*UDPConn, error) { - return dialUDP(net, laddr, raddr, noDeadline) -} - -func dialUDP(net string, laddr, raddr *UDPAddr, deadline time.Time) (*UDPConn, error) { - if !deadline.IsZero() { - panic("net.dialUDP: deadline not implemented on Plan 9") - } - switch net { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} - } - if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} - } - fd, err := dialPlan9(net, laddr, raddr) +func dialUDP(ctx context.Context, net string, laddr, raddr *UDPAddr) (*UDPConn, error) { + fd, err := dialPlan9(ctx, net, laddr, raddr) if err != nil { return nil, err } @@ -167,49 +91,23 @@ func unmarshalUDPHeader(b []byte) (*udpHeader, []byte) { return h, b } -// ListenUDP listens for incoming UDP packets addressed to the local -// address laddr. Net must be "udp", "udp4", or "udp6". If laddr has -// a port of 0, ListenUDP will choose an available port. -// The LocalAddr method of the returned UDPConn can be used to -// discover the port. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send UDP packets with per-packet -// addressing. -func ListenUDP(net string, laddr *UDPAddr) (*UDPConn, error) { - switch net { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - laddr = &UDPAddr{} - } - l, err := listenPlan9(net, laddr) +func listenUDP(ctx context.Context, network string, laddr *UDPAddr) (*UDPConn, error) { + l, err := listenPlan9(ctx, network, laddr) if err != nil { return nil, err } _, err = l.ctl.WriteString("headers") if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } l.data, err = os.OpenFile(l.dir+"/data", os.O_RDWR, 0) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } fd, err := l.netFD() return newUDPConn(fd), err } -// ListenMulticastUDP listens for incoming multicast UDP packets -// addressed to the group address gaddr on the interface ifi. -// Network must be "udp", "udp4" or "udp6". -// ListenMulticastUDP uses the system-assigned multicast interface -// when ifi is nil, although this is not recommended because the -// assignment depends on platforms and sometimes it might require -// routing configuration. -// -// ListenMulticastUDP is just for convenience of simple, small -// applications. There are golang.org/x/net/ipv4 and -// golang.org/x/net/ipv6 packages for general purpose uses. -func ListenMulticastUDP(network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: syscall.EPLAN9} +func listenMulticastUDP(ctx context.Context, network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + return nil, syscall.EPLAN9 } diff --git a/libgo/go/net/udpsock_posix.go b/libgo/go/net/udpsock_posix.go index 932c6ce..4924801 100644 --- a/libgo/go/net/udpsock_posix.go +++ b/libgo/go/net/udpsock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,8 +7,8 @@ package net import ( + "context" "syscall" - "time" ) func sockaddrToUDP(sa syscall.Sockaddr) Addr { @@ -38,25 +38,7 @@ func (a *UDPAddr) sockaddr(family int) (syscall.Sockaddr, error) { return ipToSockaddr(family, a.IP, a.Port, a.Zone) } -// UDPConn is the implementation of the Conn and PacketConn interfaces -// for UDP network connections. -type UDPConn struct { - conn -} - -func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } - -// ReadFromUDP reads a UDP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *UDPConn) ReadFromUDP(b []byte) (int, *UDPAddr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } +func (c *UDPConn) readFrom(b []byte) (int, *UDPAddr, error) { var addr *UDPAddr n, sa, err := c.fd.readFrom(b) switch sa := sa.(type) { @@ -65,33 +47,10 @@ func (c *UDPConn) ReadFromUDP(b []byte) (int, *UDPAddr, error) { case *syscall.SockaddrInet6: addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return n, addr, err -} - -// ReadFrom implements the PacketConn ReadFrom method. -func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } - n, addr, err := c.ReadFromUDP(b) - if addr == nil { - return n, nil, err - } return n, addr, err } -// ReadMsgUDP reads a packet from c, copying the payload into b and -// the associated out-of-band data into oob. It returns the number -// of bytes copied into b, the number of bytes copied into oob, the -// flags that were set on the packet and the source address of the -// packet. -func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { - if !c.ok() { - return 0, 0, 0, nil, syscall.EINVAL - } +func (c *UDPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { var sa syscall.Sockaddr n, oobn, flags, sa, err = c.fd.readMsg(b, oob) switch sa := sa.(type) { @@ -100,159 +59,68 @@ func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, case *syscall.SockaddrInet6: addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return } -// WriteToUDP writes a UDP packet to addr via c, copying the payload -// from b. -// -// WriteToUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } +func (c *UDPConn) writeTo(b []byte, addr *UDPAddr) (int, error) { if c.fd.isConnected { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, ErrWriteToConnected } if addr == nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} + return 0, errMissingAddress } sa, err := addr.sockaddr(c.fd.family) if err != nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - n, err := c.fd.writeTo(b, sa) - if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + return 0, err } - return n, err + return c.fd.writeTo(b, sa) } -// WriteTo implements the PacketConn WriteTo method. -func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } - a, ok := addr.(*UDPAddr) - if !ok { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} - } - return c.WriteToUDP(b, a) -} - -// WriteMsgUDP writes a packet to addr via c if c isn't connected, or -// to c's remote destination address if c is connected (in which case -// addr must be nil). The payload is copied from b and the associated -// out-of-band data is copied from oob. It returns the number of -// payload and out-of-band bytes written. -func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { - if !c.ok() { - return 0, 0, syscall.EINVAL - } +func (c *UDPConn) writeMsg(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { if c.fd.isConnected && addr != nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, 0, ErrWriteToConnected } if !c.fd.isConnected && addr == nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: errMissingAddress} + return 0, 0, errMissingAddress } - var sa syscall.Sockaddr - sa, err = addr.sockaddr(c.fd.family) - if err != nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - n, oobn, err = c.fd.writeMsg(b, oob, sa) + sa, err := addr.sockaddr(c.fd.family) if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + return 0, 0, err } - return -} - -// DialUDP connects to the remote address raddr on the network net, -// which must be "udp", "udp4", or "udp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialUDP(net string, laddr, raddr *UDPAddr) (*UDPConn, error) { - switch net { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} - } - if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} - } - return dialUDP(net, laddr, raddr, noDeadline) + return c.fd.writeMsg(b, oob, sa) } -func dialUDP(net string, laddr, raddr *UDPAddr, deadline time.Time) (*UDPConn, error) { - fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_DGRAM, 0, "dial", noCancel) +func dialUDP(ctx context.Context, net string, laddr, raddr *UDPAddr) (*UDPConn, error) { + fd, err := internetSocket(ctx, net, laddr, raddr, syscall.SOCK_DGRAM, 0, "dial") if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, err } return newUDPConn(fd), nil } -// ListenUDP listens for incoming UDP packets addressed to the local -// address laddr. Net must be "udp", "udp4", or "udp6". If laddr has -// a port of 0, ListenUDP will choose an available port. -// The LocalAddr method of the returned UDPConn can be used to -// discover the port. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send UDP packets with per-packet -// addressing. -func ListenUDP(net string, laddr *UDPAddr) (*UDPConn, error) { - switch net { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - laddr = &UDPAddr{} - } - fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_DGRAM, 0, "listen", noCancel) +func listenUDP(ctx context.Context, network string, laddr *UDPAddr) (*UDPConn, error) { + fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_DGRAM, 0, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } return newUDPConn(fd), nil } -// ListenMulticastUDP listens for incoming multicast UDP packets -// addressed to the group address gaddr on the interface ifi. -// Network must be "udp", "udp4" or "udp6". -// ListenMulticastUDP uses the system-assigned multicast interface -// when ifi is nil, although this is not recommended because the -// assignment depends on platforms and sometimes it might require -// routing configuration. -// -// ListenMulticastUDP is just for convenience of simple, small -// applications. There are golang.org/x/net/ipv4 and -// golang.org/x/net/ipv6 packages for general purpose uses. -func ListenMulticastUDP(network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { - switch network { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: UnknownNetworkError(network)} - } - if gaddr == nil || gaddr.IP == nil { - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: errMissingAddress} - } - fd, err := internetSocket(network, gaddr, nil, noDeadline, syscall.SOCK_DGRAM, 0, "listen", noCancel) +func listenMulticastUDP(ctx context.Context, network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + fd, err := internetSocket(ctx, network, gaddr, nil, syscall.SOCK_DGRAM, 0, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr, Err: err} + return nil, err } c := newUDPConn(fd) if ip4 := gaddr.IP.To4(); ip4 != nil { if err := listenIPv4MulticastUDP(c, ifi, ip4); err != nil { c.Close() - return nil, &OpError{Op: "listen", Net: network, Source: c.fd.laddr, Addr: &IPAddr{IP: ip4}, Err: err} + return nil, err } } else { if err := listenIPv6MulticastUDP(c, ifi, gaddr.IP); err != nil { c.Close() - return nil, &OpError{Op: "listen", Net: network, Source: c.fd.laddr, Addr: &IPAddr{IP: gaddr.IP}, Err: err} + return nil, err } } return c, nil diff --git a/libgo/go/net/udp_test.go b/libgo/go/net/udpsock_test.go index b25f96a..29d769c 100644 --- a/libgo/go/net/udp_test.go +++ b/libgo/go/net/udpsock_test.go @@ -1,16 +1,54 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "internal/testenv" "reflect" "runtime" "testing" "time" ) +func BenchmarkUDP6LinkLocalUnicast(b *testing.B) { + testHookUninstaller.Do(uninstallTestHooks) + + if !supportsIPv6 { + b.Skip("IPv6 is not supported") + } + ifi := loopbackInterface() + if ifi == nil { + b.Skip("loopback interface not found") + } + lla := ipv6LinkLocalUnicastAddr(ifi) + if lla == "" { + b.Skip("IPv6 link-local unicast address not found") + } + + c1, err := ListenPacket("udp6", JoinHostPort(lla+"%"+ifi.Name, "0")) + if err != nil { + b.Fatal(err) + } + defer c1.Close() + c2, err := ListenPacket("udp6", JoinHostPort(lla+"%"+ifi.Name, "0")) + if err != nil { + b.Fatal(err) + } + defer c2.Close() + + var buf [1]byte + for i := 0; i < b.N; i++ { + if _, err := c1.WriteTo(buf[:], c2.LocalAddr()); err != nil { + b.Fatal(err) + } + if _, _, err := c2.ReadFrom(buf[:]); err != nil { + b.Fatal(err) + } + } +} + type resolveUDPAddrTest struct { network string litAddrOrName string @@ -178,9 +216,7 @@ var udpConnLocalNameTests = []struct { } func TestUDPConnLocalName(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) for _, tt := range udpConnLocalNameTests { c, err := ListenUDP(tt.net, tt.laddr) @@ -234,9 +270,8 @@ func TestUDPConnLocalAndRemoteNames(t *testing.T) { } func TestIPv6LinkLocalUnicastUDP(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv6 { t.Skip("IPv6 is not supported") } @@ -356,7 +391,7 @@ func TestUDPZeroByteBuffer(t *testing.T) { switch err { case nil: // ReadFrom succeeds default: // Read may timeout, it depends on the platform - if nerr, ok := err.(Error); (!ok || !nerr.Timeout()) && runtime.GOOS != "windows" { // Windows retruns WSAEMSGSIZ + if nerr, ok := err.(Error); (!ok || !nerr.Timeout()) && runtime.GOOS != "windows" { // Windows returns WSAEMSGSIZ t.Fatal(err) } } diff --git a/libgo/go/net/unixsock.go b/libgo/go/net/unixsock.go index eb91d0d..bacdaa4 100644 --- a/libgo/go/net/unixsock.go +++ b/libgo/go/net/unixsock.go @@ -1,9 +1,16 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import ( + "context" + "os" + "syscall" + "time" +) + // UnixAddr represents the address of a Unix domain socket end point. type UnixAddr struct { Name string @@ -45,3 +52,268 @@ func ResolveUnixAddr(net, addr string) (*UnixAddr, error) { return nil, UnknownNetworkError(net) } } + +// UnixConn is an implementation of the Conn interface for connections +// to Unix domain sockets. +type UnixConn struct { + conn +} + +// CloseRead shuts down the reading side of the Unix domain connection. +// Most callers should just use Close. +func (c *UnixConn) CloseRead() error { + if !c.ok() { + return syscall.EINVAL + } + if err := c.fd.closeRead(); err != nil { + return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// CloseWrite shuts down the writing side of the Unix domain connection. +// Most callers should just use Close. +func (c *UnixConn) CloseWrite() error { + if !c.ok() { + return syscall.EINVAL + } + if err := c.fd.closeWrite(); err != nil { + return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// ReadFromUnix reads a packet from c, copying the payload into b. It +// returns the number of bytes copied into b and the source address of +// the packet. +// +// ReadFromUnix can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, addr, err +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + if addr == nil { + return n, nil, err + } + return n, addr, err +} + +// ReadMsgUnix reads a packet from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number of +// bytes copied into b, the number of bytes copied into oob, the flags +// that were set on the packet, and the source address of the packet. +func (c *UnixConn) ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { + if !c.ok() { + return 0, 0, 0, nil, syscall.EINVAL + } + n, oobn, flags, addr, err = c.readMsg(b, oob) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return +} + +// WriteToUnix writes a packet to addr via c, copying the payload from b. +// +// WriteToUnix can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.writeTo(b, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return n, err +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *UnixConn) WriteTo(b []byte, addr Addr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*UnixAddr) + if !ok { + return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} + } + n, err := c.writeTo(b, a) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: a.opAddr(), Err: err} + } + return n, err +} + +// WriteMsgUnix writes a packet to addr via c, copying the payload +// from b and the associated out-of-band data from oob. It returns +// the number of payload and out-of-band bytes written. +func (c *UnixConn) WriteMsgUnix(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { + if !c.ok() { + return 0, 0, syscall.EINVAL + } + n, oobn, err = c.writeMsg(b, oob, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return +} + +func newUnixConn(fd *netFD) *UnixConn { return &UnixConn{conn{fd}} } + +// DialUnix connects to the remote address raddr on the network net, +// which must be "unix", "unixgram" or "unixpacket". If laddr is not +// nil, it is used as the local address for the connection. +func DialUnix(net string, laddr, raddr *UnixAddr) (*UnixConn, error) { + switch net { + case "unix", "unixgram", "unixpacket": + default: + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + } + c, err := dialUnix(context.Background(), net, laddr, raddr) + if err != nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + } + return c, nil +} + +// UnixListener is a Unix domain socket listener. Clients should +// typically use variables of type Listener instead of assuming Unix +// domain sockets. +type UnixListener struct { + fd *netFD + path string + unlink bool +} + +func (ln *UnixListener) ok() bool { return ln != nil && ln.fd != nil } + +// AcceptUnix accepts the next incoming call and returns the new +// connection. +func (l *UnixListener) AcceptUnix() (*UnixConn, error) { + if !l.ok() { + return nil, syscall.EINVAL + } + c, err := l.accept() + if err != nil { + return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return c, nil +} + +// Accept implements the Accept method in the Listener interface. +// Returned connections will be of type *UnixConn. +func (l *UnixListener) Accept() (Conn, error) { + if !l.ok() { + return nil, syscall.EINVAL + } + c, err := l.accept() + if err != nil { + return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return c, nil +} + +// Close stops listening on the Unix address. Already accepted +// connections are not closed. +func (l *UnixListener) Close() error { + if !l.ok() { + return syscall.EINVAL + } + if err := l.close(); err != nil { + return &OpError{Op: "close", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return nil +} + +// Addr returns the listener's network address. +// The Addr returned is shared by all invocations of Addr, so +// do not modify it. +func (l *UnixListener) Addr() Addr { return l.fd.laddr } + +// SetDeadline sets the deadline associated with the listener. +// A zero time value disables the deadline. +func (l *UnixListener) SetDeadline(t time.Time) error { + if !l.ok() { + return syscall.EINVAL + } + if err := l.fd.setDeadline(t); err != nil { + return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return nil +} + +// File returns a copy of the underlying os.File, set to blocking +// mode. It is the caller's responsibility to close f when finished. +// Closing l does not affect f, and closing f does not affect l. +// +// The returned os.File's file descriptor is different from the +// connection's. Attempting to change properties of the original +// using this duplicate may or may not have the desired effect. +func (l *UnixListener) File() (f *os.File, err error) { + if !l.ok() { + return nil, syscall.EINVAL + } + f, err = l.file() + if err != nil { + err = &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return +} + +// ListenUnix announces on the Unix domain socket laddr and returns a +// Unix listener. The network net must be "unix" or "unixpacket". +func ListenUnix(net string, laddr *UnixAddr) (*UnixListener, error) { + switch net { + case "unix", "unixpacket": + default: + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + } + if laddr == nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: errMissingAddress} + } + ln, err := listenUnix(context.Background(), net, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return ln, nil +} + +// ListenUnixgram listens for incoming Unix datagram packets addressed +// to the local address laddr. The network net must be "unixgram". +// The returned connection's ReadFrom and WriteTo methods can be used +// to receive and send packets with per-packet addressing. +func ListenUnixgram(net string, laddr *UnixAddr) (*UnixConn, error) { + switch net { + case "unixgram": + default: + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + } + if laddr == nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: errMissingAddress} + } + c, err := listenUnixgram(context.Background(), net, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return c, nil +} diff --git a/libgo/go/net/unixsock_plan9.go b/libgo/go/net/unixsock_plan9.go index 84b6b60..e70eb21 100644 --- a/libgo/go/net/unixsock_plan9.go +++ b/libgo/go/net/unixsock_plan9.go @@ -1,147 +1,51 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "os" "syscall" - "time" ) -// UnixConn is an implementation of the Conn interface for connections -// to Unix domain sockets. -type UnixConn struct { - conn +func (c *UnixConn) readFrom(b []byte) (int, *UnixAddr, error) { + return 0, nil, syscall.EPLAN9 } -// ReadFromUnix reads a packet from c, copying the payload into b. It -// returns the number of bytes copied into b and the source address of -// the packet. -// -// ReadFromUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *UnixConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { + return 0, 0, 0, nil, syscall.EPLAN9 } -// ReadFrom implements the PacketConn ReadFrom method. -func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *UnixConn) writeTo(b []byte, addr *UnixAddr) (int, error) { + return 0, syscall.EPLAN9 } -// ReadMsgUnix reads a packet from c, copying the payload into b and -// the associated out-of-band data into oob. It returns the number of -// bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet, and the source address of the packet. -func (c *UnixConn) ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { - return 0, 0, 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *UnixConn) writeMsg(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { + return 0, 0, syscall.EPLAN9 } -// WriteToUnix writes a packet to addr via c, copying the payload from b. -// -// WriteToUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (int, error) { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} +func dialUnix(ctx context.Context, network string, laddr, raddr *UnixAddr) (*UnixConn, error) { + return nil, syscall.EPLAN9 } -// WriteTo implements the PacketConn WriteTo method. -func (c *UnixConn) WriteTo(b []byte, addr Addr) (int, error) { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr, Err: syscall.EPLAN9} +func (ln *UnixListener) accept() (*UnixConn, error) { + return nil, syscall.EPLAN9 } -// WriteMsgUnix writes a packet to addr via c, copying the payload -// from b and the associated out-of-band data from oob. It returns -// the number of payload and out-of-band bytes written. -func (c *UnixConn) WriteMsgUnix(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { - return 0, 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} +func (ln *UnixListener) close() error { + return syscall.EPLAN9 } -// CloseRead shuts down the reading side of the Unix domain connection. -// Most callers should just use Close. -func (c *UnixConn) CloseRead() error { - return &OpError{Op: "close", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (ln *UnixListener) file() (*os.File, error) { + return nil, syscall.EPLAN9 } -// CloseWrite shuts down the writing side of the Unix domain connection. -// Most callers should just use Close. -func (c *UnixConn) CloseWrite() error { - return &OpError{Op: "close", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func listenUnix(ctx context.Context, network string, laddr *UnixAddr) (*UnixListener, error) { + return nil, syscall.EPLAN9 } -// DialUnix connects to the remote address raddr on the network net, -// which must be "unix", "unixgram" or "unixpacket". If laddr is not -// nil, it is used as the local address for the connection. -func DialUnix(net string, laddr, raddr *UnixAddr) (*UnixConn, error) { - return dialUnix(net, laddr, raddr, noDeadline) -} - -func dialUnix(net string, laddr, raddr *UnixAddr, deadline time.Time) (*UnixConn, error) { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: syscall.EPLAN9} -} - -// UnixListener is a Unix domain socket listener. Clients should -// typically use variables of type Listener instead of assuming Unix -// domain sockets. -type UnixListener struct { - fd *netFD -} - -// ListenUnix announces on the Unix domain socket laddr and returns a -// Unix listener. The network net must be "unix" or "unixpacket". -func ListenUnix(net string, laddr *UnixAddr) (*UnixListener, error) { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: syscall.EPLAN9} -} - -// AcceptUnix accepts the next incoming call and returns the new -// connection. -func (l *UnixListener) AcceptUnix() (*UnixConn, error) { - return nil, &OpError{Op: "accept", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// Accept implements the Accept method in the Listener interface; it -// waits for the next call and returns a generic Conn. -func (l *UnixListener) Accept() (Conn, error) { - return nil, &OpError{Op: "accept", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// Close stops listening on the Unix address. Already accepted -// connections are not closed. -func (l *UnixListener) Close() error { - return &OpError{Op: "close", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// Addr returns the listener's network address. -// The Addr returned is shared by all invocations of Addr, so -// do not modify it. -func (l *UnixListener) Addr() Addr { return nil } - -// SetDeadline sets the deadline associated with the listener. -// A zero time value disables the deadline. -func (l *UnixListener) SetDeadline(t time.Time) error { - return &OpError{Op: "set", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// File returns a copy of the underlying os.File, set to blocking -// mode. It is the caller's responsibility to close f when finished. -// Closing l does not affect f, and closing f does not affect l. -// -// The returned os.File's file descriptor is different from the -// connection's. Attempting to change properties of the original -// using this duplicate may or may not have the desired effect. -func (l *UnixListener) File() (*os.File, error) { - return nil, &OpError{Op: "file", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// ListenUnixgram listens for incoming Unix datagram packets addressed -// to the local address laddr. The network net must be "unixgram". -// The returned connection's ReadFrom and WriteTo methods can be used -// to receive and send packets with per-packet addressing. -func ListenUnixgram(net string, laddr *UnixAddr) (*UnixConn, error) { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: syscall.EPLAN9} +func listenUnixgram(ctx context.Context, network string, laddr *UnixAddr) (*UnixConn, error) { + return nil, syscall.EPLAN9 } diff --git a/libgo/go/net/unixsock_posix.go b/libgo/go/net/unixsock_posix.go index fb2397e..5f0999c 100644 --- a/libgo/go/net/unixsock_posix.go +++ b/libgo/go/net/unixsock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,13 +7,13 @@ package net import ( + "context" "errors" "os" "syscall" - "time" ) -func unixSocket(net string, laddr, raddr sockaddr, mode string, deadline time.Time) (*netFD, error) { +func unixSocket(ctx context.Context, net string, laddr, raddr sockaddr, mode string) (*netFD, error) { var sotype int switch net { case "unix": @@ -42,7 +42,7 @@ func unixSocket(net string, laddr, raddr sockaddr, mode string, deadline time.Ti return nil, errors.New("unknown mode: " + mode) } - fd, err := socket(net, syscall.AF_UNIX, sotype, 0, false, laddr, raddr, deadline, noCancel) + fd, err := socket(ctx, net, syscall.AF_UNIX, sotype, 0, false, laddr, raddr) if err != nil { return nil, err } @@ -94,25 +94,7 @@ func (a *UnixAddr) sockaddr(family int) (syscall.Sockaddr, error) { return &syscall.SockaddrUnix{Name: a.Name}, nil } -// UnixConn is an implementation of the Conn interface for connections -// to Unix domain sockets. -type UnixConn struct { - conn -} - -func newUnixConn(fd *netFD) *UnixConn { return &UnixConn{conn{fd}} } - -// ReadFromUnix reads a packet from c, copying the payload into b. It -// returns the number of bytes copied into b and the source address of -// the packet. -// -// ReadFromUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } +func (c *UnixConn) readFrom(b []byte) (int, *UnixAddr, error) { var addr *UnixAddr n, sa, err := c.fd.readFrom(b) switch sa := sa.(type) { @@ -121,211 +103,66 @@ func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { addr = &UnixAddr{Name: sa.Name, Net: sotypeToNet(c.fd.sotype)} } } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return n, addr, err -} - -// ReadFrom implements the PacketConn ReadFrom method. -func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } - n, addr, err := c.ReadFromUnix(b) - if addr == nil { - return n, nil, err - } return n, addr, err } -// ReadMsgUnix reads a packet from c, copying the payload into b and -// the associated out-of-band data into oob. It returns the number of -// bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet, and the source address of the packet. -func (c *UnixConn) ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { - if !c.ok() { - return 0, 0, 0, nil, syscall.EINVAL - } - n, oobn, flags, sa, err := c.fd.readMsg(b, oob) +func (c *UnixConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { + var sa syscall.Sockaddr + n, oobn, flags, sa, err = c.fd.readMsg(b, oob) switch sa := sa.(type) { case *syscall.SockaddrUnix: if sa.Name != "" { addr = &UnixAddr{Name: sa.Name, Net: sotypeToNet(c.fd.sotype)} } } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return } -// WriteToUnix writes a packet to addr via c, copying the payload from b. -// -// WriteToUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } +func (c *UnixConn) writeTo(b []byte, addr *UnixAddr) (int, error) { if c.fd.isConnected { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, ErrWriteToConnected } if addr == nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} + return 0, errMissingAddress } if addr.Net != sotypeToNet(c.fd.sotype) { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EAFNOSUPPORT} + return 0, syscall.EAFNOSUPPORT } sa := &syscall.SockaddrUnix{Name: addr.Name} - n, err := c.fd.writeTo(b, sa) - if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - return n, err -} - -// WriteTo implements the PacketConn WriteTo method. -func (c *UnixConn) WriteTo(b []byte, addr Addr) (n int, err error) { - if !c.ok() { - return 0, syscall.EINVAL - } - a, ok := addr.(*UnixAddr) - if !ok { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} - } - return c.WriteToUnix(b, a) + return c.fd.writeTo(b, sa) } -// WriteMsgUnix writes a packet to addr via c, copying the payload -// from b and the associated out-of-band data from oob. It returns -// the number of payload and out-of-band bytes written. -func (c *UnixConn) WriteMsgUnix(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { - if !c.ok() { - return 0, 0, syscall.EINVAL - } +func (c *UnixConn) writeMsg(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { if c.fd.sotype == syscall.SOCK_DGRAM && c.fd.isConnected { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, 0, ErrWriteToConnected } var sa syscall.Sockaddr if addr != nil { if addr.Net != sotypeToNet(c.fd.sotype) { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EAFNOSUPPORT} + return 0, 0, syscall.EAFNOSUPPORT } sa = &syscall.SockaddrUnix{Name: addr.Name} } - n, oobn, err = c.fd.writeMsg(b, oob, sa) - if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - return -} - -// CloseRead shuts down the reading side of the Unix domain connection. -// Most callers should just use Close. -func (c *UnixConn) CloseRead() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeRead() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err + return c.fd.writeMsg(b, oob, sa) } -// CloseWrite shuts down the writing side of the Unix domain connection. -// Most callers should just use Close. -func (c *UnixConn) CloseWrite() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeWrite() +func dialUnix(ctx context.Context, net string, laddr, raddr *UnixAddr) (*UnixConn, error) { + fd, err := unixSocket(ctx, net, laddr, raddr, "dial") if err != nil { - err = &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err -} - -// DialUnix connects to the remote address raddr on the network net, -// which must be "unix", "unixgram" or "unixpacket". If laddr is not -// nil, it is used as the local address for the connection. -func DialUnix(net string, laddr, raddr *UnixAddr) (*UnixConn, error) { - switch net { - case "unix", "unixgram", "unixpacket": - default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} - } - return dialUnix(net, laddr, raddr, noDeadline) -} - -func dialUnix(net string, laddr, raddr *UnixAddr, deadline time.Time) (*UnixConn, error) { - fd, err := unixSocket(net, laddr, raddr, "dial", deadline) - if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} - } - return newUnixConn(fd), nil -} - -// UnixListener is a Unix domain socket listener. Clients should -// typically use variables of type Listener instead of assuming Unix -// domain sockets. -type UnixListener struct { - fd *netFD - path string - unlink bool -} - -// ListenUnix announces on the Unix domain socket laddr and returns a -// Unix listener. The network net must be "unix" or "unixpacket". -func ListenUnix(net string, laddr *UnixAddr) (*UnixListener, error) { - switch net { - case "unix", "unixpacket": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: errMissingAddress} - } - fd, err := unixSocket(net, laddr, nil, "listen", noDeadline) - if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} - } - return &UnixListener{fd: fd, path: fd.laddr.String(), unlink: true}, nil -} - -// AcceptUnix accepts the next incoming call and returns the new -// connection. -func (l *UnixListener) AcceptUnix() (*UnixConn, error) { - if l == nil || l.fd == nil { - return nil, syscall.EINVAL - } - fd, err := l.fd.accept() - if err != nil { - return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + return nil, err } return newUnixConn(fd), nil } -// Accept implements the Accept method in the Listener interface; it -// waits for the next call and returns a generic Conn. -func (l *UnixListener) Accept() (c Conn, err error) { - c1, err := l.AcceptUnix() +func (ln *UnixListener) accept() (*UnixConn, error) { + fd, err := ln.fd.accept() if err != nil { return nil, err } - return c1, nil + return newUnixConn(fd), nil } -// Close stops listening on the Unix address. Already accepted -// connections are not closed. -func (l *UnixListener) Close() error { - if l == nil || l.fd == nil { - return syscall.EINVAL - } - +func (ln *UnixListener) close() error { // The operating system doesn't clean up // the file that announcing created, so // we have to clean it up ourselves. @@ -334,66 +171,34 @@ func (l *UnixListener) Close() error { // and replaced our socket name already-- // but this sequence (remove then close) // is at least compatible with the auto-remove - // sequence in ListenUnix. It's only non-Go + // sequence in ListenUnix. It's only non-Go // programs that can mess us up. - if l.path[0] != '@' && l.unlink { - syscall.Unlink(l.path) - } - err := l.fd.Close() - if err != nil { - err = &OpError{Op: "close", Net: l.fd.net, Source: l.fd.laddr, Addr: l.fd.raddr, Err: err} + if ln.path[0] != '@' && ln.unlink { + syscall.Unlink(ln.path) } - return err + return ln.fd.Close() } -// Addr returns the listener's network address. -// The Addr returned is shared by all invocations of Addr, so -// do not modify it. -func (l *UnixListener) Addr() Addr { return l.fd.laddr } - -// SetDeadline sets the deadline associated with the listener. -// A zero time value disables the deadline. -func (l *UnixListener) SetDeadline(t time.Time) error { - if l == nil || l.fd == nil { - return syscall.EINVAL - } - if err := l.fd.setDeadline(t); err != nil { - return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} +func (ln *UnixListener) file() (*os.File, error) { + f, err := ln.fd.dup() + if err != nil { + return nil, err } - return nil + return f, nil } -// File returns a copy of the underlying os.File, set to blocking -// mode. It is the caller's responsibility to close f when finished. -// Closing l does not affect f, and closing f does not affect l. -// -// The returned os.File's file descriptor is different from the -// connection's. Attempting to change properties of the original -// using this duplicate may or may not have the desired effect. -func (l *UnixListener) File() (f *os.File, err error) { - f, err = l.fd.dup() +func listenUnix(ctx context.Context, network string, laddr *UnixAddr) (*UnixListener, error) { + fd, err := unixSocket(ctx, network, laddr, nil, "listen") if err != nil { - err = &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + return nil, err } - return + return &UnixListener{fd: fd, path: fd.laddr.String(), unlink: true}, nil } -// ListenUnixgram listens for incoming Unix datagram packets addressed -// to the local address laddr. The network net must be "unixgram". -// The returned connection's ReadFrom and WriteTo methods can be used -// to receive and send packets with per-packet addressing. -func ListenUnixgram(net string, laddr *UnixAddr) (*UnixConn, error) { - switch net { - case "unixgram": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: errMissingAddress} - } - fd, err := unixSocket(net, laddr, nil, "listen", noDeadline) +func listenUnixgram(ctx context.Context, network string, laddr *UnixAddr) (*UnixConn, error) { + fd, err := unixSocket(ctx, network, laddr, nil, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, err } return newUnixConn(fd), nil } diff --git a/libgo/go/net/unix_test.go b/libgo/go/net/unixsock_test.go index f0c5830..f0f88ed 100644 --- a/libgo/go/net/unix_test.go +++ b/libgo/go/net/unixsock_test.go @@ -1,4 +1,4 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -8,6 +8,7 @@ package net import ( "bytes" + "internal/testenv" "os" "reflect" "runtime" @@ -20,6 +21,9 @@ func TestReadUnixgramWithUnnamedSocket(t *testing.T) { if !testableNetwork("unixgram") { t.Skip("unixgram test") } + if runtime.GOOS == "openbsd" { + testenv.SkipFlaky(t, 15157) + } addr := testUnixAddr() la, err := ResolveUnixAddr("unixgram", addr) @@ -440,34 +444,3 @@ func TestUnixUnlink(t *testing.T) { t.Fatal("closing unix listener did not remove unix socket") } } - -// forceGoDNS forces the resolver configuration to use the pure Go resolver -// and returns a fixup function to restore the old settings. -func forceGoDNS() func() { - c := systemConf() - oldGo := c.netGo - oldCgo := c.netCgo - fixup := func() { - c.netGo = oldGo - c.netCgo = oldCgo - } - c.netGo = true - c.netCgo = false - return fixup -} - -// forceCgoDNS forces the resolver configuration to use the cgo resolver -// and returns a fixup function to restore the old settings. -// (On non-Unix systems forceCgoDNS returns nil.) -func forceCgoDNS() func() { - c := systemConf() - oldGo := c.netGo - oldCgo := c.netCgo - fixup := func() { - c.netGo = oldGo - c.netCgo = oldCgo - } - c.netGo = false - c.netCgo = true - return fixup -} diff --git a/libgo/go/net/url/url.go b/libgo/go/net/url/url.go index 1a93e34..30e9277 100644 --- a/libgo/go/net/url/url.go +++ b/libgo/go/net/url/url.go @@ -3,9 +3,13 @@ // license that can be found in the LICENSE file. // Package url parses URLs and implements query escaping. -// See RFC 3986. package url +// See RFC 3986. This package generally follows RFC 3986, except where +// it deviates for compatibility reasons. When sending changes, first +// search old issues for history on decisions. Unit tests should also +// contain references to issue numbers with details. + import ( "bytes" "errors" @@ -307,14 +311,15 @@ func escape(s string, mode encoding) string { // construct a URL struct directly and set the Opaque field instead of Path. // These still work as well. type URL struct { - Scheme string - Opaque string // encoded opaque data - User *Userinfo // username and password information - Host string // host or host:port - Path string - RawPath string // encoded path hint (Go 1.5 and later only; see EscapedPath method) - RawQuery string // encoded query values, without '?' - Fragment string // fragment for references, without '#' + Scheme string + Opaque string // encoded opaque data + User *Userinfo // username and password information + Host string // host or host:port + Path string + RawPath string // encoded path hint (Go 1.5 and later only; see EscapedPath method) + ForceQuery bool // append a query ('?') even if RawQuery is empty + RawQuery string // encoded query values, without '?' + Fragment string // fragment for references, without '#' } // User returns a Userinfo containing the provided username @@ -410,10 +415,11 @@ func split(s string, c string, cutc bool) (string, string) { // Parse parses rawurl into a URL structure. // The rawurl may be relative or absolute. -func Parse(rawurl string) (url *URL, err error) { +func Parse(rawurl string) (*URL, error) { // Cut off #frag u, frag := split(rawurl, "#", true) - if url, err = parse(u, false); err != nil { + url, err := parse(u, false) + if err != nil { return nil, err } if frag == "" { @@ -425,16 +431,16 @@ func Parse(rawurl string) (url *URL, err error) { return url, nil } -// ParseRequestURI parses rawurl into a URL structure. It assumes that +// ParseRequestURI parses rawurl into a URL structure. It assumes that // rawurl was received in an HTTP request, so the rawurl is interpreted // only as an absolute URI or an absolute path. // The string rawurl is assumed not to have a #fragment suffix. // (Web browsers strip #fragment before sending the URL to a web server.) -func ParseRequestURI(rawurl string) (url *URL, err error) { +func ParseRequestURI(rawurl string) (*URL, error) { return parse(rawurl, true) } -// parse parses a URL from a string in one of two contexts. If +// parse parses a URL from a string in one of two contexts. If // viaRequest is true, the URL is assumed to have arrived via an HTTP request, // in which case only absolute URLs or path-absolute relative URLs are allowed. // If viaRequest is false, all forms of relative URLs are allowed. @@ -459,7 +465,12 @@ func parse(rawurl string, viaRequest bool) (url *URL, err error) { } url.Scheme = strings.ToLower(url.Scheme) - rest, url.RawQuery = split(rest, "?", true) + if strings.HasSuffix(rest, "?") && strings.Count(rest, "?") == 1 { + url.ForceQuery = true + rest = rest[:len(rest)-1] + } else { + rest, url.RawQuery = split(rest, "?", true) + } if !strings.HasPrefix(rest, "/") { if url.Scheme != "" { @@ -511,7 +522,7 @@ func parseAuthority(authority string) (user *Userinfo, host string, err error) { return nil, host, nil } userinfo := authority[:i] - if strings.Index(userinfo, ":") < 0 { + if !strings.Contains(userinfo, ":") { if userinfo, err = unescape(userinfo, encodeUserPassword); err != nil { return nil, "", err } @@ -684,7 +695,7 @@ func (u *URL) String() string { } buf.WriteString(path) } - if u.RawQuery != "" { + if u.ForceQuery || u.RawQuery != "" { buf.WriteByte('?') buf.WriteString(u.RawQuery) } @@ -709,8 +720,8 @@ func (v Values) Get(key string) string { if v == nil { return "" } - vs, ok := v[key] - if !ok || len(vs) == 0 { + vs := v[key] + if len(vs) == 0 { return "" } return vs[0] @@ -738,10 +749,10 @@ func (v Values) Del(key string) { // ParseQuery always returns a non-nil map containing all the // valid query parameters found; err describes the first decoding error // encountered, if any. -func ParseQuery(query string) (m Values, err error) { - m = make(Values) - err = parseQuery(m, query) - return +func ParseQuery(query string) (Values, error) { + m := make(Values) + err := parseQuery(m, query) + return m, err } func parseQuery(m Values, query string) (err error) { @@ -845,8 +856,8 @@ func (u *URL) IsAbs() bool { return u.Scheme != "" } -// Parse parses a URL in the context of the receiver. The provided URL -// may be relative or absolute. Parse returns nil, err on parse +// Parse parses a URL in the context of the receiver. The provided URL +// may be relative or absolute. Parse returns nil, err on parse // failure, otherwise its return value is the same as ResolveReference. func (u *URL) Parse(ref string) (*URL, error) { refurl, err := Parse(ref) @@ -858,7 +869,7 @@ func (u *URL) Parse(ref string) (*URL, error) { // ResolveReference resolves a URI reference to an absolute URI from // an absolute base URI, per RFC 3986 Section 5.2. The URI reference -// may be relative or absolute. ResolveReference always returns a new +// may be relative or absolute. ResolveReference always returns a new // URL instance, even if the returned URL is identical to either the // base or reference. If ref is an absolute URL, then ResolveReference // ignores base and returns a copy of ref. @@ -913,7 +924,7 @@ func (u *URL) RequestURI() string { result = u.Scheme + ":" + result } } - if u.RawQuery != "" { + if u.ForceQuery || u.RawQuery != "" { result += "?" + u.RawQuery } return result diff --git a/libgo/go/net/url/url_test.go b/libgo/go/net/url/url_test.go index d3f8487..7560f22 100644 --- a/libgo/go/net/url/url_test.go +++ b/libgo/go/net/url/url_test.go @@ -72,6 +72,28 @@ var urltests = []URLTest{ }, "ftp://john%20doe@www.google.com/", }, + // empty query + { + "http://www.google.com/?", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + ForceQuery: true, + }, + "", + }, + // query ending in question mark (Issue 14573) + { + "http://www.google.com/?foo=bar?", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + RawQuery: "foo=bar?", + }, + "", + }, // query { "http://www.google.com/?q=go+language", @@ -553,8 +575,8 @@ func ufmt(u *URL) string { pass = p } } - return fmt.Sprintf("opaque=%q, scheme=%q, user=%#v, pass=%#v, host=%q, path=%q, rawpath=%q, rawq=%q, frag=%q", - u.Opaque, u.Scheme, user, pass, u.Host, u.Path, u.RawPath, u.RawQuery, u.Fragment) + return fmt.Sprintf("opaque=%q, scheme=%q, user=%#v, pass=%#v, host=%q, path=%q, rawpath=%q, rawq=%q, frag=%q, forcequery=%v", + u.Opaque, u.Scheme, user, pass, u.Host, u.Path, u.RawPath, u.RawQuery, u.Fragment, u.ForceQuery) } func DoTest(t *testing.T, parse func(string) (*URL, error), name string, tests []URLTest) { @@ -589,7 +611,7 @@ func BenchmarkString(b *testing.B) { g = u.String() } b.StopTimer() - if w := tt.roundtrip; g != w { + if w := tt.roundtrip; b.N > 0 && g != w { b.Errorf("Parse(%q).String() == %q, want %q", tt.in, g, w) } } @@ -874,11 +896,13 @@ var resolveReferenceTests = []struct { // Absolute URL references {"http://foo.com?a=b", "https://bar.com/", "https://bar.com/"}, {"http://foo.com/", "https://bar.com/?a=b", "https://bar.com/?a=b"}, + {"http://foo.com/", "https://bar.com/?", "https://bar.com/?"}, {"http://foo.com/bar", "mailto:foo@example.com", "mailto:foo@example.com"}, // Path-absolute references {"http://foo.com/bar", "/baz", "http://foo.com/baz"}, {"http://foo.com/bar?a=b#f", "/baz", "http://foo.com/baz"}, + {"http://foo.com/bar?a=b", "/baz?", "http://foo.com/baz?"}, {"http://foo.com/bar?a=b", "/baz?c=d", "http://foo.com/baz?c=d"}, // Scheme-relative @@ -1217,6 +1241,15 @@ var requritests = []RequestURITest{ }, "//foo", }, + { + &URL{ + Scheme: "http", + Host: "example.com", + Path: "/foo", + ForceQuery: true, + }, + "/foo?", + }, } func TestRequestURI(t *testing.T) { |