diff options
author | Ian Lance Taylor <iant@golang.org> | 2017-09-14 17:11:35 +0000 |
---|---|---|
committer | Ian Lance Taylor <ian@gcc.gnu.org> | 2017-09-14 17:11:35 +0000 |
commit | bc998d034f45d1828a8663b2eed928faf22a7d01 (patch) | |
tree | 8d262a22ca7318f4bcd64269fe8fe9e45bcf8d0f /libgo/go/encoding | |
parent | a41a6142df74219f596e612d3a7775f68ca6e96f (diff) | |
download | gcc-bc998d034f45d1828a8663b2eed928faf22a7d01.zip gcc-bc998d034f45d1828a8663b2eed928faf22a7d01.tar.gz gcc-bc998d034f45d1828a8663b2eed928faf22a7d01.tar.bz2 |
libgo: update to go1.9
Reviewed-on: https://go-review.googlesource.com/63753
From-SVN: r252767
Diffstat (limited to 'libgo/go/encoding')
40 files changed, 1565 insertions, 598 deletions
diff --git a/libgo/go/encoding/ascii85/ascii85_test.go b/libgo/go/encoding/ascii85/ascii85_test.go index aad199b..1a3a87a 100644 --- a/libgo/go/encoding/ascii85/ascii85_test.go +++ b/libgo/go/encoding/ascii85/ascii85_test.go @@ -16,6 +16,18 @@ type testpair struct { decoded, encoded string } +var bigtest = testpair{ + "Man is distinguished, not only by his reason, but by this singular passion from " + + "other animals, which is a lust of the mind, that by a perseverance of delight in " + + "the continued and indefatigable generation of knowledge, exceeds the short " + + "vehemence of any carnal pleasure.", + "9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKF<GL>Cj@.4Gp$d7F!,L7@<6@)/0JDEF<G%<+EV:2F!,\n" + + "O<DJ+*.@<*K0@<6L(Df-\\0Ec5e;DffZ(EZee.Bl.9pF\"AGXBPCsi+DGm>@3BB/F*&OCAfu2/AKY\n" + + "i(DIb:@FD,*)+C]U=@3BN#EcYf8ATD3s@q?d$AftVqCh[NqF<G:8+EV:.+Cf>-FD5W8ARlolDIa\n" + + "l(DId<j@<?3r@:F%a+D58'ATD4$Bl@l3De:,-DJs`8ARoFb/0JMK@qB4^F!,R<AKZ&-DfTqBG%G\n" + + ">uD.RTpAKYo'+CT/5+Cei#DII?(E,9)oF*2M7/c\n", +} + var pairs = []testpair{ // Encode returns 0 when len(src) is 0 { @@ -23,17 +35,7 @@ var pairs = []testpair{ "", }, // Wikipedia example - { - "Man is distinguished, not only by his reason, but by this singular passion from " + - "other animals, which is a lust of the mind, that by a perseverance of delight in " + - "the continued and indefatigable generation of knowledge, exceeds the short " + - "vehemence of any carnal pleasure.", - "9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKF<GL>Cj@.4Gp$d7F!,L7@<6@)/0JDEF<G%<+EV:2F!,\n" + - "O<DJ+*.@<*K0@<6L(Df-\\0Ec5e;DffZ(EZee.Bl.9pF\"AGXBPCsi+DGm>@3BB/F*&OCAfu2/AKY\n" + - "i(DIb:@FD,*)+C]U=@3BN#EcYf8ATD3s@q?d$AftVqCh[NqF<G:8+EV:.+Cf>-FD5W8ARlolDIa\n" + - "l(DId<j@<?3r@:F%a+D58'ATD4$Bl@l3De:,-DJs`8ARoFb/0JMK@qB4^F!,R<AKZ&-DfTqBG%G\n" + - ">uD.RTpAKYo'+CT/5+Cei#DII?(E,9)oF*2M7/c\n", - }, + bigtest, // Special case when shortening !!!!! to z. { "\000\000\000\000", @@ -41,9 +43,8 @@ var pairs = []testpair{ }, } -var bigtest = pairs[len(pairs)-1] - func testEqual(t *testing.T, msg string, args ...interface{}) bool { + t.Helper() if args[len(args)-2] != args[len(args)-1] { t.Errorf(msg, args...) return false @@ -134,11 +135,15 @@ func TestDecoderBuffering(t *testing.T) { decoder := NewDecoder(strings.NewReader(bigtest.encoded)) buf := make([]byte, len(bigtest.decoded)+12) var total int - for total = 0; total < len(bigtest.decoded); { - n, err := decoder.Read(buf[total : total+bs]) - testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, error(nil)) + var n int + var err error + for total = 0; total < len(bigtest.decoded) && err == nil; { + n, err = decoder.Read(buf[total : total+bs]) total += n } + if err != nil && err != io.EOF { + t.Errorf("Read from %q at pos %d = %d, unexpected error %v", bigtest.encoded, total, n, err) + } testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) } } diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go index 044f74a..b8e2770 100644 --- a/libgo/go/encoding/asn1/asn1.go +++ b/libgo/go/encoding/asn1/asn1.go @@ -22,6 +22,7 @@ package asn1 import ( "errors" "fmt" + "math" "math/big" "reflect" "strconv" @@ -206,6 +207,14 @@ func parseBitString(bytes []byte) (ret BitString, err error) { return } +// NULL + +// NullRawValue is a RawValue with its Tag set to the ASN.1 NULL type tag (5). +var NullRawValue = RawValue{Tag: TagNull} + +// NullBytes contains bytes representing the DER-encoded ASN.1 NULL type. +var NullBytes = []byte{TagNull, 0} + // OBJECT IDENTIFIER // An ObjectIdentifier represents an ASN.1 OBJECT IDENTIFIER. @@ -293,16 +302,24 @@ type Flag bool // given byte slice. It returns the value and the new offset. func parseBase128Int(bytes []byte, initOffset int) (ret, offset int, err error) { offset = initOffset + var ret64 int64 for shifted := 0; offset < len(bytes); shifted++ { - if shifted == 4 { + // 5 * 7 bits per byte == 35 bits of data + // Thus the representation is either non-minimal or too large for an int32 + if shifted == 5 { err = StructuralError{"base 128 integer too large"} return } - ret <<= 7 + ret64 <<= 7 b := bytes[offset] - ret |= int(b & 0x7f) + ret64 |= int64(b & 0x7f) offset++ if b&0x80 == 0 { + ret = int(ret64) + // Ensure that the returned value fits in an int on all platforms + if ret64 > math.MaxInt32 { + err = StructuralError{"base 128 integer too large"} + } return } } @@ -975,12 +992,12 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { // // The following tags on struct fields have special meaning to Unmarshal: // -// application specifies that a APPLICATION tag is used -// default:x sets the default value for optional integer fields (only used if optional is also present) -// explicit specifies that an additional, explicit tag wraps the implicit one -// optional marks the field as ASN.1 OPTIONAL -// set causes a SET, rather than a SEQUENCE type to be expected -// tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC +// application specifies that a APPLICATION tag is used +// default:x sets the default value for optional integer fields (only used if optional is also present) +// explicit specifies that an additional, explicit tag wraps the implicit one +// optional marks the field as ASN.1 OPTIONAL +// set causes a SET, rather than a SEQUENCE type to be expected +// tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC // // If the type of the first field of a structure is RawContent then the raw // ASN1 contents of the struct will be stored in it. diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go index 9976656..c9eda40 100644 --- a/libgo/go/encoding/asn1/asn1_test.go +++ b/libgo/go/encoding/asn1/asn1_test.go @@ -7,6 +7,7 @@ package asn1 import ( "bytes" "fmt" + "math" "math/big" "reflect" "strings" @@ -386,6 +387,8 @@ var tagAndLengthData = []tagAndLengthTest{ {[]byte{0xa0, 0x81, 0x7f}, false, tagAndLength{}}, // Tag numbers which would overflow int32 are rejected. (The value below is 2^31.) {[]byte{0x1f, 0x88, 0x80, 0x80, 0x80, 0x00, 0x00}, false, tagAndLength{}}, + // Tag numbers that fit in an int32 are valid. (The value below is 2^31 - 1.) + {[]byte{0x1f, 0x87, 0xFF, 0xFF, 0xFF, 0x7F, 0x00}, true, tagAndLength{tag: math.MaxInt32}}, // Long tag number form may not be used for tags that fit in short form. {[]byte{0x1f, 0x1e, 0x00}, false, tagAndLength{}}, } @@ -476,6 +479,7 @@ var unmarshalTestData = []struct { out interface{} }{ {[]byte{0x02, 0x01, 0x42}, newInt(0x42)}, + {[]byte{0x05, 0x00}, &RawValue{0, 5, false, []byte{}, []byte{0x05, 0x00}}}, {[]byte{0x30, 0x08, 0x06, 0x06, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d}, &TestObjectIdentifierStruct{[]int{1, 2, 840, 113549}}}, {[]byte{0x03, 0x04, 0x06, 0x6e, 0x5d, 0xc0}, &BitString{[]byte{110, 93, 192}, 18}}, {[]byte{0x30, 0x09, 0x02, 0x01, 0x01, 0x02, 0x01, 0x02, 0x02, 0x01, 0x03}, &[]int{1, 2, 3}}, @@ -1004,3 +1008,28 @@ func TestUnexportedStructField(t *testing.T) { t.Errorf("got %v, want %v", err, want) } } + +func TestNull(t *testing.T) { + marshaled, err := Marshal(NullRawValue) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(NullBytes, marshaled) { + t.Errorf("Expected Marshal of NullRawValue to yeild %x, got %x", NullBytes, marshaled) + } + + unmarshaled := RawValue{} + if _, err := Unmarshal(NullBytes, &unmarshaled); err != nil { + t.Fatal(err) + } + + unmarshaled.FullBytes = NullRawValue.FullBytes + if len(unmarshaled.Bytes) == 0 { + // DeepEqual considers a nil slice and an empty slice to be different. + unmarshaled.Bytes = NullRawValue.Bytes + } + + if !reflect.DeepEqual(NullRawValue, unmarshaled) { + t.Errorf("Expected Unmarshal of NullBytes to yield %v, got %v", NullRawValue, unmarshaled) + } +} diff --git a/libgo/go/encoding/asn1/common.go b/libgo/go/encoding/asn1/common.go index 0695180..cd93b27 100644 --- a/libgo/go/encoding/asn1/common.go +++ b/libgo/go/encoding/asn1/common.go @@ -24,6 +24,7 @@ const ( TagInteger = 2 TagBitString = 3 TagOctetString = 4 + TagNull = 5 TagOID = 6 TagEnum = 10 TagUTF8String = 12 diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index 225fd08..fdadb39 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -643,10 +643,12 @@ func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { // In addition to the struct tags recognised by Unmarshal, the following can be // used: // -// ia5: causes strings to be marshaled as ASN.1, IA5 strings -// omitempty: causes empty slices to be skipped -// printable: causes strings to be marshaled as ASN.1, PrintableString strings. -// utf8: causes strings to be marshaled as ASN.1, UTF8 strings +// ia5: causes strings to be marshaled as ASN.1, IA5String values +// omitempty: causes empty slices to be skipped +// printable: causes strings to be marshaled as ASN.1, PrintableString values +// utf8: causes strings to be marshaled as ASN.1, UTF8String values +// utc: causes time.Time to be marshaled as ASN.1, UTCTime values +// generalized: causes time.Time to be marshaled as ASN.1, GeneralizedTime values func Marshal(val interface{}) ([]byte, error) { e, err := makeField(reflect.ValueOf(val), fieldParameters{}) if err != nil { diff --git a/libgo/go/encoding/base32/base32.go b/libgo/go/encoding/base32/base32.go index c193e65..bf341b5 100644 --- a/libgo/go/encoding/base32/base32.go +++ b/libgo/go/encoding/base32/base32.go @@ -23,8 +23,14 @@ import ( type Encoding struct { encode string decodeMap [256]byte + padChar rune } +const ( + StdPadding rune = '=' // Standard padding character + NoPadding rune = -1 // No padding +) + const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" const encodeHex = "0123456789ABCDEFGHIJKLMNOPQRSTUV" @@ -33,6 +39,8 @@ const encodeHex = "0123456789ABCDEFGHIJKLMNOPQRSTUV" func NewEncoding(encoder string) *Encoding { e := new(Encoding) e.encode = encoder + e.padChar = StdPadding + for i := 0; i < len(e.decodeMap); i++ { e.decodeMap[i] = 0xFF } @@ -57,6 +65,26 @@ var removeNewlinesMapper = func(r rune) rune { return r } +// WithPadding creates a new encoding identical to enc except +// with a specified padding character, or NoPadding to disable padding. +// The padding character must not be '\r' or '\n', must not +// be contained in the encoding's alphabet and must be a rune equal or +// below '\xff'. +func (enc Encoding) WithPadding(padding rune) *Encoding { + if padding == '\r' || padding == '\n' || padding > 0xff { + panic("invalid padding") + } + + for i := 0; i < len(enc.encode); i++ { + if rune(enc.encode[i]) == padding { + panic("padding contained in alphabet") + } + } + + enc.padChar = padding + return &enc +} + /* * Encoder */ @@ -73,60 +101,63 @@ func (enc *Encoding) Encode(dst, src []byte) { } for len(src) > 0 { - var b0, b1, b2, b3, b4, b5, b6, b7 byte + var b [8]byte // Unpack 8x 5-bit source blocks into a 5 byte // destination quantum switch len(src) { default: - b7 = src[4] & 0x1F - b6 = src[4] >> 5 + b[7] = src[4] & 0x1F + b[6] = src[4] >> 5 fallthrough case 4: - b6 |= (src[3] << 3) & 0x1F - b5 = (src[3] >> 2) & 0x1F - b4 = src[3] >> 7 + b[6] |= (src[3] << 3) & 0x1F + b[5] = (src[3] >> 2) & 0x1F + b[4] = src[3] >> 7 fallthrough case 3: - b4 |= (src[2] << 1) & 0x1F - b3 = (src[2] >> 4) & 0x1F + b[4] |= (src[2] << 1) & 0x1F + b[3] = (src[2] >> 4) & 0x1F fallthrough case 2: - b3 |= (src[1] << 4) & 0x1F - b2 = (src[1] >> 1) & 0x1F - b1 = (src[1] >> 6) & 0x1F + b[3] |= (src[1] << 4) & 0x1F + b[2] = (src[1] >> 1) & 0x1F + b[1] = (src[1] >> 6) & 0x1F fallthrough case 1: - b1 |= (src[0] << 2) & 0x1F - b0 = src[0] >> 3 + b[1] |= (src[0] << 2) & 0x1F + b[0] = src[0] >> 3 } // Encode 5-bit blocks using the base32 alphabet - dst[0] = enc.encode[b0] - dst[1] = enc.encode[b1] - dst[2] = enc.encode[b2] - dst[3] = enc.encode[b3] - dst[4] = enc.encode[b4] - dst[5] = enc.encode[b5] - dst[6] = enc.encode[b6] - dst[7] = enc.encode[b7] + for i := 0; i < 8; i++ { + if len(dst) > i { + dst[i] = enc.encode[b[i]] + } + } // Pad the final quantum if len(src) < 5 { - dst[7] = '=' + if enc.padChar == NoPadding { + break + } + + dst[7] = byte(enc.padChar) if len(src) < 4 { - dst[6] = '=' - dst[5] = '=' + dst[6] = byte(enc.padChar) + dst[5] = byte(enc.padChar) if len(src) < 3 { - dst[4] = '=' + dst[4] = byte(enc.padChar) if len(src) < 2 { - dst[3] = '=' - dst[2] = '=' + dst[3] = byte(enc.padChar) + dst[2] = byte(enc.padChar) } } } + break } + src = src[5:] dst = dst[8:] } @@ -219,7 +250,12 @@ func NewEncoder(enc *Encoding, w io.Writer) io.WriteCloser { // EncodedLen returns the length in bytes of the base32 encoding // of an input buffer of length n. -func (enc *Encoding) EncodedLen(n int) int { return (n + 4) / 5 * 8 } +func (enc *Encoding) EncodedLen(n int) int { + if enc.padChar == NoPadding { + return (n*8 + 4) / 5 + } + return (n + 4) / 5 * 8 +} /* * Decoder @@ -243,19 +279,28 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { dlen := 8 for j := 0; j < 8; { - if len(src) == 0 { + + // We have reached the end and are missing padding + if len(src) == 0 && enc.padChar != NoPadding { return n, false, CorruptInputError(olen - len(src) - j) } + + // We have reached the end and are not expecing any padding + if len(src) == 0 && enc.padChar == NoPadding { + dlen, end = j, true + break + } + in := src[0] src = src[1:] - if in == '=' && j >= 2 && len(src) < 8 { + if in == byte(enc.padChar) && j >= 2 && len(src) < 8 { // We've reached the end and there's padding if len(src)+j < 8-1 { // not enough padding return n, false, CorruptInputError(olen) } for k := 0; k < 8-1-j; k++ { - if len(src) > k && src[k] != '=' { + if len(src) > k && src[k] != byte(enc.padChar) { // incorrect padding return n, false, CorruptInputError(olen - len(src) + k - 1) } @@ -296,7 +341,11 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { case 2: dst[0] = dbuf[0]<<3 | dbuf[1]>>2 } - dst = dst[5:] + + if !end { + dst = dst[5:] + } + switch dlen { case 2: n += 1 @@ -343,18 +392,33 @@ type decoder struct { outbuf [1024 / 8 * 5]byte } -func (d *decoder) Read(p []byte) (n int, err error) { - if d.err != nil { - return 0, d.err +func readEncodedData(r io.Reader, buf []byte, min int) (n int, err error) { + for n < min && err == nil { + var nn int + nn, err = r.Read(buf[n:]) + n += nn } + if n < min && n > 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + } + return +} +func (d *decoder) Read(p []byte) (n int, err error) { // Use leftover decoded output from last read. if len(d.out) > 0 { n = copy(p, d.out) d.out = d.out[n:] + if len(d.out) == 0 { + return n, d.err + } return n, nil } + if d.err != nil { + return 0, d.err + } + // Read a chunk. nn := len(p) / 5 * 8 if nn < 8 { @@ -363,7 +427,8 @@ func (d *decoder) Read(p []byte) (n int, err error) { if nn > len(d.buf) { nn = len(d.buf) } - nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 8-d.nbuf) + + nn, d.err = readEncodedData(d.r, d.buf[d.nbuf:nn], 8-d.nbuf) d.nbuf += nn if d.nbuf < 8 { return 0, d.err @@ -373,21 +438,30 @@ func (d *decoder) Read(p []byte) (n int, err error) { nr := d.nbuf / 8 * 8 nw := d.nbuf / 8 * 5 if nw > len(p) { - nw, d.end, d.err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) + nw, d.end, err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) d.out = d.outbuf[0:nw] n = copy(p, d.out) d.out = d.out[n:] } else { - n, d.end, d.err = d.enc.decode(p, d.buf[0:nr]) + n, d.end, err = d.enc.decode(p, d.buf[0:nr]) } d.nbuf -= nr for i := 0; i < d.nbuf; i++ { d.buf[i] = d.buf[i+nr] } - if d.err == nil { + if err != nil && (d.err == nil || d.err == io.EOF) { d.err = err } + + if len(d.out) > 0 { + // We cannot return all the decoded bytes to the caller in this + // invocation of Read, so we return a nil error to ensure that Read + // will be called again. The error stored in d.err, if any, will be + // returned with the last set of decoded bytes. + return n, nil + } + return n, d.err } @@ -407,7 +481,7 @@ func (r *newlineFilteringReader) Read(p []byte) (int, error) { offset++ } } - if offset > 0 { + if err != nil || offset > 0 { return offset, err } // Previous buffer entirely whitespace, read again @@ -423,4 +497,10 @@ func NewDecoder(enc *Encoding, r io.Reader) io.Reader { // DecodedLen returns the maximum length in bytes of the decoded data // corresponding to n bytes of base32-encoded data. -func (enc *Encoding) DecodedLen(n int) int { return n / 8 * 5 } +func (enc *Encoding) DecodedLen(n int) int { + if enc.padChar == NoPadding { + return n * 5 / 8 + } + + return n / 8 * 5 +} diff --git a/libgo/go/encoding/base32/base32_test.go b/libgo/go/encoding/base32/base32_test.go index 66a48a3..56b229d 100644 --- a/libgo/go/encoding/base32/base32_test.go +++ b/libgo/go/encoding/base32/base32_test.go @@ -6,6 +6,7 @@ package base32 import ( "bytes" + "errors" "io" "io/ioutil" "strings" @@ -43,6 +44,7 @@ var bigtest = testpair{ } func testEqual(t *testing.T, msg string, args ...interface{}) bool { + t.Helper() if args[len(args)-2] != args[len(args)-1] { t.Errorf(msg, args...) return false @@ -123,16 +125,174 @@ func TestDecoder(t *testing.T) { } } +type badReader struct { + data []byte + errs []error + called int + limit int +} + +// Populates p with data, returns a count of the bytes written and an +// error. The error returned is taken from badReader.errs, with each +// invocation of Read returning the next error in this slice, or io.EOF, +// if all errors from the slice have already been returned. The +// number of bytes returned is determined by the size of the input buffer +// the test passes to decoder.Read and will be a multiple of 8, unless +// badReader.limit is non zero. +func (b *badReader) Read(p []byte) (int, error) { + lim := len(p) + if b.limit != 0 && b.limit < lim { + lim = b.limit + } + if len(b.data) < lim { + lim = len(b.data) + } + for i := range p[:lim] { + p[i] = b.data[i] + } + b.data = b.data[lim:] + err := io.EOF + if b.called < len(b.errs) { + err = b.errs[b.called] + } + b.called++ + return lim, err +} + +// TestIssue20044 tests that decoder.Read behaves correctly when the caller +// supplied reader returns an error. +func TestIssue20044(t *testing.T) { + badErr := errors.New("bad reader error") + testCases := []struct { + r badReader + res string + err error + dbuflen int + }{ + // Check valid input data accompanied by an error is processed and the error is propagated. + {r: badReader{data: []byte("MY======"), errs: []error{badErr}}, + res: "f", err: badErr}, + // Check a read error accompanied by input data consisting of newlines only is propagated. + {r: badReader{data: []byte("\n\n\n\n\n\n\n\n"), errs: []error{badErr, nil}}, + res: "", err: badErr}, + // Reader will be called twice. The first time it will return 8 newline characters. The + // second time valid base32 encoded data and an error. The data should be decoded + // correctly and the error should be propagated. + {r: badReader{data: []byte("\n\n\n\n\n\n\n\nMY======"), errs: []error{nil, badErr}}, + res: "f", err: badErr, dbuflen: 8}, + // Reader returns invalid input data (too short) and an error. Verify the reader + // error is returned. + {r: badReader{data: []byte("MY====="), errs: []error{badErr}}, + res: "", err: badErr}, + // Reader returns invalid input data (too short) but no error. Verify io.ErrUnexpectedEOF + // is returned. + {r: badReader{data: []byte("MY====="), errs: []error{nil}}, + res: "", err: io.ErrUnexpectedEOF}, + // Reader returns invalid input data and an error. Verify the reader and not the + // decoder error is returned. + {r: badReader{data: []byte("Ma======"), errs: []error{badErr}}, + res: "", err: badErr}, + // Reader returns valid data and io.EOF. Check data is decoded and io.EOF is propagated. + {r: badReader{data: []byte("MZXW6YTB"), errs: []error{io.EOF}}, + res: "fooba", err: io.EOF}, + // Check errors are properly reported when decoder.Read is called multiple times. + // decoder.Read will be called 8 times, badReader.Read will be called twice, returning + // valid data both times but an error on the second call. + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{nil, badErr}}, + res: "leasure.", err: badErr, dbuflen: 1}, + // Check io.EOF is properly reported when decoder.Read is called multiple times. + // decoder.Read will be called 8 times, badReader.Read will be called twice, returning + // valid data both times but io.EOF on the second call. + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{nil, io.EOF}}, + res: "leasure.", err: io.EOF, dbuflen: 1}, + // The following two test cases check that errors are propagated correctly when more than + // 8 bytes are read at a time. + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{io.EOF}}, + res: "leasure.", err: io.EOF, dbuflen: 11}, + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{badErr}}, + res: "leasure.", err: badErr, dbuflen: 11}, + // Check that errors are correctly propagated when the reader returns valid bytes in + // groups that are not divisible by 8. The first read will return 11 bytes and no + // error. The second will return 7 and an error. The data should be decoded correctly + // and the error should be propagated. + {r: badReader{data: []byte("NRSWC43VOJSS4==="), errs: []error{nil, badErr}, limit: 11}, + res: "leasure.", err: badErr}, + } + + for _, tc := range testCases { + input := tc.r.data + decoder := NewDecoder(StdEncoding, &tc.r) + var dbuflen int + if tc.dbuflen > 0 { + dbuflen = tc.dbuflen + } else { + dbuflen = StdEncoding.DecodedLen(len(input)) + } + dbuf := make([]byte, dbuflen) + var err error + var res []byte + for err == nil { + var n int + n, err = decoder.Read(dbuf) + if n > 0 { + res = append(res, dbuf[:n]...) + } + } + + testEqual(t, "Decoding of %q = %q, want %q", string(input), string(res), tc.res) + testEqual(t, "Decoding of %q err = %v, expected %v", string(input), err, tc.err) + } +} + +// TestDecoderError verifies decode errors are propagated when there are no read +// errors. +func TestDecoderError(t *testing.T) { + for _, readErr := range []error{io.EOF, nil} { + input := "MZXW6YTb" + dbuf := make([]byte, StdEncoding.DecodedLen(len(input))) + br := badReader{data: []byte(input), errs: []error{readErr}} + decoder := NewDecoder(StdEncoding, &br) + n, err := decoder.Read(dbuf) + testEqual(t, "Read after EOF, n = %d, expected %d", n, 0) + if _, ok := err.(CorruptInputError); !ok { + t.Errorf("Corrupt input error expected. Found %T", err) + } + } +} + +// TestReaderEOF ensures decoder.Read behaves correctly when input data is +// exhausted. +func TestReaderEOF(t *testing.T) { + for _, readErr := range []error{io.EOF, nil} { + input := "MZXW6YTB" + br := badReader{data: []byte(input), errs: []error{nil, readErr}} + decoder := NewDecoder(StdEncoding, &br) + dbuf := make([]byte, StdEncoding.DecodedLen(len(input))) + n, err := decoder.Read(dbuf) + testEqual(t, "Decoding of %q err = %v, expected %v", string(input), err, error(nil)) + n, err = decoder.Read(dbuf) + testEqual(t, "Read after EOF, n = %d, expected %d", n, 0) + testEqual(t, "Read after EOF, err = %v, expected %v", err, io.EOF) + n, err = decoder.Read(dbuf) + testEqual(t, "Read after EOF, n = %d, expected %d", n, 0) + testEqual(t, "Read after EOF, err = %v, expected %v", err, io.EOF) + } +} + func TestDecoderBuffering(t *testing.T) { for bs := 1; bs <= 12; bs++ { decoder := NewDecoder(StdEncoding, strings.NewReader(bigtest.encoded)) buf := make([]byte, len(bigtest.decoded)+12) var total int - for total = 0; total < len(bigtest.decoded); { - n, err := decoder.Read(buf[total : total+bs]) - testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, error(nil)) + var n int + var err error + for total = 0; total < len(bigtest.decoded) && err == nil; { + n, err = decoder.Read(buf[total : total+bs]) total += n } + if err != nil && err != io.EOF { + t.Errorf("Read from %q at pos %d = %d, unexpected error %v", bigtest.encoded, total, n, err) + } testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) } } @@ -300,3 +460,121 @@ func BenchmarkDecodeString(b *testing.B) { StdEncoding.DecodeString(data) } } + +func TestWithCustomPadding(t *testing.T) { + for _, testcase := range pairs { + defaultPadding := StdEncoding.EncodeToString([]byte(testcase.decoded)) + customPadding := StdEncoding.WithPadding('@').EncodeToString([]byte(testcase.decoded)) + expected := strings.Replace(defaultPadding, "=", "@", -1) + + if expected != customPadding { + t.Errorf("Expected custom %s, got %s", expected, customPadding) + } + if testcase.encoded != defaultPadding { + t.Errorf("Expected %s, got %s", testcase.encoded, defaultPadding) + } + } +} + +func TestWithoutPadding(t *testing.T) { + for _, testcase := range pairs { + defaultPadding := StdEncoding.EncodeToString([]byte(testcase.decoded)) + customPadding := StdEncoding.WithPadding(NoPadding).EncodeToString([]byte(testcase.decoded)) + expected := strings.TrimRight(defaultPadding, "=") + + if expected != customPadding { + t.Errorf("Expected custom %s, got %s", expected, customPadding) + } + if testcase.encoded != defaultPadding { + t.Errorf("Expected %s, got %s", testcase.encoded, defaultPadding) + } + } +} + +func TestDecodeWithPadding(t *testing.T) { + encodings := []*Encoding{ + StdEncoding, + StdEncoding.WithPadding('-'), + StdEncoding.WithPadding(NoPadding), + } + + for i, enc := range encodings { + for _, pair := range pairs { + + input := pair.decoded + encoded := enc.EncodeToString([]byte(input)) + + decoded, err := enc.DecodeString(encoded) + if err != nil { + t.Errorf("DecodeString Error for encoding %d (%q): %v", i, input, err) + } + + if input != string(decoded) { + t.Errorf("Unexpected result for encoding %d: got %q; want %q", i, decoded, input) + } + } + } +} + +func TestDecodeWithWrongPadding(t *testing.T) { + encoded := StdEncoding.EncodeToString([]byte("foobar")) + + _, err := StdEncoding.WithPadding('-').DecodeString(encoded) + if err == nil { + t.Error("expected error") + } + + _, err = StdEncoding.WithPadding(NoPadding).DecodeString(encoded) + if err == nil { + t.Error("expected error") + } +} + +func TestEncodedDecodedLen(t *testing.T) { + type test struct { + in int + wantEnc int + wantDec int + } + data := bytes.Repeat([]byte("x"), 100) + for _, test := range []struct { + name string + enc *Encoding + cases []test + }{ + {"StdEncoding", StdEncoding, []test{ + {0, 0, 0}, + {1, 8, 5}, + {5, 8, 5}, + {6, 16, 10}, + {10, 16, 10}, + }}, + {"NoPadding", StdEncoding.WithPadding(NoPadding), []test{ + {0, 0, 0}, + {1, 2, 1}, + {2, 4, 2}, + {5, 8, 5}, + {6, 10, 6}, + {7, 12, 7}, + {10, 16, 10}, + {11, 18, 11}, + }}, + } { + t.Run(test.name, func(t *testing.T) { + for _, tc := range test.cases { + encLen := test.enc.EncodedLen(tc.in) + decLen := test.enc.DecodedLen(encLen) + enc := test.enc.EncodeToString(data[:tc.in]) + if len(enc) != encLen { + t.Fatalf("EncodedLen(%d) = %d but encoded to %q (%d)", tc.in, encLen, enc, len(enc)) + } + if encLen != tc.wantEnc { + t.Fatalf("EncodedLen(%d) = %d; want %d", tc.in, encLen, tc.wantEnc) + } + if decLen != tc.wantDec { + t.Fatalf("DecodedLen(%d) = %d; want %d", encLen, decLen, tc.wantDec) + } + } + }) + } +} diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go index d2efad4..b208f9e 100644 --- a/libgo/go/encoding/base64/base64.go +++ b/libgo/go/encoding/base64/base64.go @@ -35,13 +35,19 @@ const encodeStd = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz012345678 const encodeURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" // NewEncoding returns a new padded Encoding defined by the given alphabet, -// which must be a 64-byte string. +// which must be a 64-byte string that does not contain the padding character +// or CR / LF ('\r', '\n'). // The resulting Encoding uses the default padding character ('='), // which may be changed or disabled via WithPadding. func NewEncoding(encoder string) *Encoding { if len(encoder) != 64 { panic("encoding alphabet is not 64-bytes long") } + for i := 0; i < len(encoder); i++ { + if encoder[i] == '\n' || encoder[i] == '\r' { + panic("encoding alphabet contains newline character") + } + } e := new(Encoding) e.padChar = StdPadding @@ -58,7 +64,20 @@ func NewEncoding(encoder string) *Encoding { // WithPadding creates a new encoding identical to enc except // with a specified padding character, or NoPadding to disable padding. +// The padding character must not be '\r' or '\n', must not +// be contained in the encoding's alphabet and must be a rune equal or +// below '\xff'. func (enc Encoding) WithPadding(padding rune) *Encoding { + if padding == '\r' || padding == '\n' || padding > 0xff { + panic("invalid padding") + } + + for i := 0; i < len(enc.encode); i++ { + if rune(enc.encode[i]) == padding { + panic("padding contained in alphabet") + } + } + enc.padChar = padding return &enc } @@ -256,19 +275,17 @@ func (e CorruptInputError) Error() string { func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { si := 0 - // skip over newlines - for si < len(src) && (src[si] == '\n' || src[si] == '\r') { - si++ - } - for si < len(src) && !end { // Decode quantum using the base64 alphabet var dbuf [4]byte dinc, dlen := 3, 4 - for j := range dbuf { + for j := 0; j < len(dbuf); j++ { if len(src) == si { - if enc.padChar != NoPadding || j < 2 { + switch { + case j == 0: + return n, false, nil + case j == 1, enc.padChar != NoPadding: return n, false, CorruptInputError(si - j) } dinc, dlen, end = j-1, j, true @@ -277,11 +294,17 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { in := src[si] si++ - // skip over newlines - for si < len(src) && (src[si] == '\n' || src[si] == '\r') { - si++ + + out := enc.decodeMap[in] + if out != 0xFF { + dbuf[j] = out + continue } + if in == '\n' || in == '\r' { + j-- + continue + } if rune(in) == enc.padChar { // We've reached the end and there's padding switch j { @@ -290,6 +313,10 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { return n, false, CorruptInputError(si - 1) case 2: // "==" is expected, the first "=" is already consumed. + // skip over newlines + for si < len(src) && (src[si] == '\n' || src[si] == '\r') { + si++ + } if si == len(src) { // not enough padding return n, false, CorruptInputError(len(src)) @@ -300,10 +327,10 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { } si++ - // skip over newlines - for si < len(src) && (src[si] == '\n' || src[si] == '\r') { - si++ - } + } + // skip over newlines + for si < len(src) && (src[si] == '\n' || src[si] == '\r') { + si++ } if si < len(src) { // trailing garbage @@ -312,10 +339,7 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { dinc, dlen, end = 3, j, true break } - dbuf[j] = enc.decodeMap[in] - if dbuf[j] == 0xFF { - return n, false, CorruptInputError(si - 1) - } + return n, false, CorruptInputError(si - 1) } // Convert 4x 6bit source bytes into 3 bytes diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go index e2e1d59..05011fb 100644 --- a/libgo/go/encoding/base64/base64_test.go +++ b/libgo/go/encoding/base64/base64_test.go @@ -7,6 +7,7 @@ package base64 import ( "bytes" "errors" + "fmt" "io" "io/ioutil" "reflect" @@ -63,7 +64,7 @@ func rawRef(ref string) string { } // Both URL and unpadding conversions -func rawUrlRef(ref string) string { +func rawURLRef(ref string) string { return rawRef(urlRef(ref)) } @@ -83,12 +84,12 @@ var encodingTests = []encodingTest{ {StdEncoding, stdRef}, {URLEncoding, urlRef}, {RawStdEncoding, rawRef}, - {RawURLEncoding, rawUrlRef}, + {RawURLEncoding, rawURLRef}, {funnyEncoding, funnyRef}, {StdEncoding.Strict(), stdRef}, {URLEncoding.Strict(), urlRef}, {RawStdEncoding.Strict(), rawRef}, - {RawURLEncoding.Strict(), rawUrlRef}, + {RawURLEncoding.Strict(), rawURLRef}, {funnyEncoding.Strict(), funnyRef}, } @@ -98,6 +99,7 @@ var bigtest = testpair{ } func testEqual(t *testing.T, msg string, args ...interface{}) bool { + t.Helper() if args[len(args)-2] != args[len(args)-1] { t.Errorf(msg, args...) return false @@ -187,11 +189,15 @@ func TestDecoderBuffering(t *testing.T) { decoder := NewDecoder(StdEncoding, strings.NewReader(bigtest.encoded)) buf := make([]byte, len(bigtest.decoded)+12) var total int - for total = 0; total < len(bigtest.decoded); { - n, err := decoder.Read(buf[total : total+bs]) - testEqual(t, "Read from %q at pos %d = %d, %v, want _, %v", bigtest.encoded, total, n, err, error(nil)) + var n int + var err error + for total = 0; total < len(bigtest.decoded) && err == nil; { + n, err = decoder.Read(buf[total : total+bs]) total += n } + if err != nil && err != io.EOF { + t.Errorf("Read from %q at pos %d = %d, unexpected error %v", bigtest.encoded, total, n, err) + } testEqual(t, "Decoding/%d of %q = %q, want %q", bs, bigtest.encoded, string(buf[0:total]), bigtest.decoded) } } @@ -202,6 +208,9 @@ func TestDecodeCorrupt(t *testing.T) { offset int // -1 means no corruption. }{ {"", -1}, + {"\n", -1}, + {"AAA=\n", -1}, + {"AAAA\n", -1}, {"!!!!", 0}, {"====", 0}, {"x===", 1}, @@ -220,6 +229,8 @@ func TestDecodeCorrupt(t *testing.T) { {"AAAA", -1}, {"AAAAAA=", 7}, {"YWJjZA=====", 8}, + {"A!\n", 1}, + {"A=\n", 1}, } for _, tc := range testCases { dbuf := make([]byte, StdEncoding.DecodedLen(len(tc.input))) @@ -466,10 +477,19 @@ func BenchmarkEncodeToString(b *testing.B) { } func BenchmarkDecodeString(b *testing.B) { - data := StdEncoding.EncodeToString(make([]byte, 8192)) - b.SetBytes(int64(len(data))) - for i := 0; i < b.N; i++ { - StdEncoding.DecodeString(data) + sizes := []int{2, 4, 8, 64, 8192} + benchFunc := func(b *testing.B, benchSize int) { + data := StdEncoding.EncodeToString(make([]byte, benchSize)) + b.SetBytes(int64(len(data))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + StdEncoding.DecodeString(data) + } + } + for _, size := range sizes { + b.Run(fmt.Sprintf("%d", size), func(b *testing.B) { + benchFunc(b, size) + }) } } diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index 3834254..2d01a3c 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -152,7 +152,8 @@ func (bigEndian) GoString() string { return "binary.BigEndian" } // When reading into structs, the field data for fields with // blank (_) field names is skipped; i.e., blank field names // may be used for padding. -// When reading into a struct, all non-blank fields must be exported. +// When reading into a struct, all non-blank fields must be exported +// or Read may panic. // // The error is EOF only if no bytes were read. // If an EOF happens after reading some but not all the bytes, diff --git a/libgo/go/encoding/binary/binary_test.go b/libgo/go/encoding/binary/binary_test.go index fc7f276..0547bee 100644 --- a/libgo/go/encoding/binary/binary_test.go +++ b/libgo/go/encoding/binary/binary_test.go @@ -500,3 +500,27 @@ func BenchmarkWriteSlice1000Int32s(b *testing.B) { } b.StopTimer() } + +func BenchmarkPutUint16(b *testing.B) { + buf := [2]byte{} + b.SetBytes(2) + for i := 0; i < b.N; i++ { + BigEndian.PutUint16(buf[:], uint16(i)) + } +} + +func BenchmarkPutUint32(b *testing.B) { + buf := [4]byte{} + b.SetBytes(4) + for i := 0; i < b.N; i++ { + BigEndian.PutUint32(buf[:], uint32(i)) + } +} + +func BenchmarkPutUint64(b *testing.B) { + buf := [8]byte{} + b.SetBytes(8) + for i := 0; i < b.N; i++ { + BigEndian.PutUint64(buf[:], uint64(i)) + } +} diff --git a/libgo/go/encoding/binary/varint.go b/libgo/go/encoding/binary/varint.go index d7a75f9..bcb8ac9 100644 --- a/libgo/go/encoding/binary/varint.go +++ b/libgo/go/encoding/binary/varint.go @@ -53,9 +53,9 @@ func PutUvarint(buf []byte, x uint64) int { // number of bytes read (> 0). If an error occurred, the value is 0 // and the number of bytes n is <= 0 meaning: // -// n == 0: buf too small -// n < 0: value larger than 64 bits (overflow) -// and -n is the number of bytes read +// n == 0: buf too small +// n < 0: value larger than 64 bits (overflow) +// and -n is the number of bytes read // func Uvarint(buf []byte) (uint64, int) { var x uint64 @@ -87,9 +87,9 @@ func PutVarint(buf []byte, x int64) int { // number of bytes read (> 0). If an error occurred, the value is 0 // and the number of bytes n is <= 0 with the following meaning: // -// n == 0: buf too small -// n < 0: value larger than 64 bits (overflow) -// and -n is the number of bytes read +// n == 0: buf too small +// n < 0: value larger than 64 bits (overflow) +// and -n is the number of bytes read // func Varint(buf []byte) (int64, int) { ux, n := Uvarint(buf) // ok to continue in presence of error diff --git a/libgo/go/encoding/csv/reader.go b/libgo/go/encoding/csv/reader.go index c8c4ca7..a3497c8 100644 --- a/libgo/go/encoding/csv/reader.go +++ b/libgo/go/encoding/csv/reader.go @@ -110,6 +110,10 @@ type Reader struct { // If TrimLeadingSpace is true, leading white space in a field is ignored. // This is done even if the field delimiter, Comma, is white space. TrimLeadingSpace bool + // ReuseRecord controls whether calls to Read may return a slice sharing + // the backing array of the previous call's returned slice for performance. + // By default, each call to Read returns newly allocated memory owned by the caller. + ReuseRecord bool line int column int @@ -122,6 +126,9 @@ type Reader struct { // Indexes of fields inside lineBuffer // The i'th field starts at offset fieldIndexes[i] in lineBuffer. fieldIndexes []int + + // only used when ReuseRecord == true + lastRecord []string } // NewReader returns a new Reader that reads from r. @@ -147,26 +154,17 @@ func (r *Reader) error(err error) error { // Except for that case, Read always returns either a non-nil // record or a non-nil error, but not both. // If there is no data left to be read, Read returns nil, io.EOF. +// If ReuseRecord is true, the returned slice may be shared +// between multiple calls to Read. func (r *Reader) Read() (record []string, err error) { - for { - record, err = r.parseRecord() - if record != nil { - break - } - if err != nil { - return nil, err - } + if r.ReuseRecord { + record, err = r.readRecord(r.lastRecord) + r.lastRecord = record + } else { + record, err = r.readRecord(nil) } - if r.FieldsPerRecord > 0 { - if len(record) != r.FieldsPerRecord { - r.column = 0 // report at start of record - return record, r.error(ErrFieldCount) - } - } else if r.FieldsPerRecord == 0 { - r.FieldsPerRecord = len(record) - } - return record, nil + return record, err } // ReadAll reads all the remaining records from r. @@ -176,7 +174,7 @@ func (r *Reader) Read() (record []string, err error) { // reported. func (r *Reader) ReadAll() (records [][]string, err error) { for { - record, err := r.Read() + record, err := r.readRecord(nil) if err == io.EOF { return records, nil } @@ -187,6 +185,31 @@ func (r *Reader) ReadAll() (records [][]string, err error) { } } +// readRecord reads and parses a single csv record from r. +// Unlike parseRecord, readRecord handles FieldsPerRecord. +// If dst has enough capacity it will be used for the returned record. +func (r *Reader) readRecord(dst []string) (record []string, err error) { + for { + record, err = r.parseRecord(dst) + if record != nil { + break + } + if err != nil { + return nil, err + } + } + + if r.FieldsPerRecord > 0 { + if len(record) != r.FieldsPerRecord { + r.column = 0 // report at start of record + return record, r.error(ErrFieldCount) + } + } else if r.FieldsPerRecord == 0 { + r.FieldsPerRecord = len(record) + } + return record, nil +} + // readRune reads one rune from r, folding \r\n to \n and keeping track // of how far into the line we have read. r.column will point to the start // of this rune, not the end of this rune. @@ -223,7 +246,8 @@ func (r *Reader) skip(delim rune) error { } // parseRecord reads and parses a single csv record from r. -func (r *Reader) parseRecord() (fields []string, err error) { +// If dst has enough capacity it will be used for the returned fields. +func (r *Reader) parseRecord(dst []string) (fields []string, err error) { // Each record starts on a new line. We increment our line // number (lines start at 1, not 0) and set column to -1 // so as we increment in readRune it points to the character we read. @@ -275,7 +299,12 @@ func (r *Reader) parseRecord() (fields []string, err error) { // minimal and a tradeoff for better performance through the combined // allocations. line := r.lineBuffer.String() - fields = make([]string, fieldCount) + + if cap(dst) >= fieldCount { + fields = dst[:fieldCount] + } else { + fields = make([]string, fieldCount) + } for i, idx := range r.fieldIndexes { if i == fieldCount-1 { diff --git a/libgo/go/encoding/csv/reader_test.go b/libgo/go/encoding/csv/reader_test.go index 7b3aca4..5ab1b61 100644 --- a/libgo/go/encoding/csv/reader_test.go +++ b/libgo/go/encoding/csv/reader_test.go @@ -24,6 +24,7 @@ var readTests = []struct { LazyQuotes bool TrailingComma bool TrimLeadingSpace bool + ReuseRecord bool Error string Line int // Expected error line if != 0 @@ -260,6 +261,15 @@ x,,, {"c", "d", "e"}, }, }, + { + Name: "ReadAllReuseRecord", + ReuseRecord: true, + Input: "a,b\nc,d", + Output: [][]string{ + {"a", "b"}, + {"c", "d"}, + }, + }, } func TestRead(t *testing.T) { @@ -274,6 +284,7 @@ func TestRead(t *testing.T) { r.LazyQuotes = tt.LazyQuotes r.TrailingComma = tt.TrailingComma r.TrimLeadingSpace = tt.TrimLeadingSpace + r.ReuseRecord = tt.ReuseRecord if tt.Comma != 0 { r.Comma = tt.Comma } @@ -369,3 +380,23 @@ xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzz xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv `, 3)) } + +func BenchmarkReadReuseRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true }, benchmarkCSVData) +} + +func BenchmarkReadReuseRecordWithFieldsPerRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true; r.FieldsPerRecord = 4 }, benchmarkCSVData) +} + +func BenchmarkReadReuseRecordWithoutFieldsPerRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true; r.FieldsPerRecord = -1 }, benchmarkCSVData) +} + +func BenchmarkReadReuseRecordLargeFields(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true }, strings.Repeat(`xxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvv +,,zzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +`, 3)) +} diff --git a/libgo/go/encoding/gob/codec_test.go b/libgo/go/encoding/gob/codec_test.go index d4002cb..eb9f306 100644 --- a/libgo/go/encoding/gob/codec_test.go +++ b/libgo/go/encoding/gob/codec_test.go @@ -47,7 +47,6 @@ func testError(t *testing.T) { if e := recover(); e != nil { t.Error(e.(gobError).err) // Will re-panic if not one of our errors, such as a runtime error. } - return } func newDecBuffer(data []byte) *decBuffer { @@ -321,7 +320,7 @@ func TestScalarEncInstructions(t *testing.T) { } } -func execDec(typ string, instr *decInstr, state *decoderState, t *testing.T, value reflect.Value) { +func execDec(instr *decInstr, state *decoderState, t *testing.T, value reflect.Value) { defer testError(t) v := int(state.decodeUint()) if v+state.fieldnum != 6 { @@ -348,7 +347,7 @@ func TestScalarDecInstructions(t *testing.T) { var data bool instr := &decInstr{decBool, 6, nil, ovfl} state := newDecodeStateFromData(boolResult) - execDec("bool", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != true { t.Errorf("bool a = %v not true", data) } @@ -358,7 +357,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int instr := &decInstr{decOpTable[reflect.Int], 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int a = %v not 17", data) } @@ -369,7 +368,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint instr := &decInstr{decOpTable[reflect.Uint], 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint a = %v not 17", data) } @@ -380,7 +379,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int8 instr := &decInstr{decInt8, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int8", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int8 a = %v not 17", data) } @@ -391,7 +390,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint8 instr := &decInstr{decUint8, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint8", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint8 a = %v not 17", data) } @@ -402,7 +401,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int16 instr := &decInstr{decInt16, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int16", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int16 a = %v not 17", data) } @@ -413,7 +412,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint16 instr := &decInstr{decUint16, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint16", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint16 a = %v not 17", data) } @@ -424,7 +423,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int32 instr := &decInstr{decInt32, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int32", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int32 a = %v not 17", data) } @@ -435,7 +434,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint32 instr := &decInstr{decUint32, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint32", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint32 a = %v not 17", data) } @@ -446,7 +445,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uintptr instr := &decInstr{decOpTable[reflect.Uintptr], 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uintptr", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uintptr a = %v not 17", data) } @@ -457,7 +456,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int64 instr := &decInstr{decInt64, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int64", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int64 a = %v not 17", data) } @@ -468,7 +467,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint64 instr := &decInstr{decUint64, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint64", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint64 a = %v not 17", data) } @@ -479,7 +478,7 @@ func TestScalarDecInstructions(t *testing.T) { var data float32 instr := &decInstr{decFloat32, 6, nil, ovfl} state := newDecodeStateFromData(floatResult) - execDec("float32", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("float32 a = %v not 17", data) } @@ -490,7 +489,7 @@ func TestScalarDecInstructions(t *testing.T) { var data float64 instr := &decInstr{decFloat64, 6, nil, ovfl} state := newDecodeStateFromData(floatResult) - execDec("float64", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("float64 a = %v not 17", data) } @@ -501,7 +500,7 @@ func TestScalarDecInstructions(t *testing.T) { var data complex64 instr := &decInstr{decOpTable[reflect.Complex64], 6, nil, ovfl} state := newDecodeStateFromData(complexResult) - execDec("complex", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17+19i { t.Errorf("complex a = %v not 17+19i", data) } @@ -512,7 +511,7 @@ func TestScalarDecInstructions(t *testing.T) { var data complex128 instr := &decInstr{decOpTable[reflect.Complex128], 6, nil, ovfl} state := newDecodeStateFromData(complexResult) - execDec("complex", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17+19i { t.Errorf("complex a = %v not 17+19i", data) } @@ -523,7 +522,7 @@ func TestScalarDecInstructions(t *testing.T) { var data []byte instr := &decInstr{decUint8Slice, 6, nil, ovfl} state := newDecodeStateFromData(bytesResult) - execDec("bytes", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if string(data) != "hello" { t.Errorf(`bytes a = %q not "hello"`, string(data)) } @@ -534,7 +533,7 @@ func TestScalarDecInstructions(t *testing.T) { var data string instr := &decInstr{decString, 6, nil, ovfl} state := newDecodeStateFromData(bytesResult) - execDec("bytes", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != "hello" { t.Errorf(`bytes a = %q not "hello"`, data) } @@ -545,11 +544,18 @@ func TestEndToEnd(t *testing.T) { type T2 struct { T string } - s1 := "string1" - s2 := "string2" + type T3 struct { + X float64 + Z *int + } type T1 struct { A, B, C int M map[string]*float64 + M2 map[int]T3 + Mstring map[string]string + Mintptr map[int]*int + Mcomp map[complex128]complex128 + Marr map[[2]string][2]*float64 EmptyMap map[string]int // to check that we receive a non-nil map. N *[3]float64 Strs *[2]string @@ -561,11 +567,35 @@ func TestEndToEnd(t *testing.T) { } pi := 3.14159 e := 2.71828 + two := 2.0 + meaning := 42 + fingers := 5 + s1 := "string1" + s2 := "string2" + var comp1 complex128 = complex(1.0, 1.0) + var comp2 complex128 = complex(1.0, 1.0) + var arr1 [2]string + arr1[0] = s1 + arr1[1] = s2 + var arr2 [2]string + arr2[0] = s2 + arr2[1] = s1 + var floatArr1 [2]*float64 + floatArr1[0] = &pi + floatArr1[1] = &e + var floatArr2 [2]*float64 + floatArr2[0] = &e + floatArr2[1] = &two t1 := &T1{ A: 17, B: 18, C: -5, M: map[string]*float64{"pi": &pi, "e": &e}, + M2: map[int]T3{4: T3{X: pi, Z: &meaning}, 10: T3{X: e, Z: &fingers}}, + Mstring: map[string]string{"pi": "3.14", "e": "2.71"}, + Mintptr: map[int]*int{meaning: &fingers, fingers: &meaning}, + Mcomp: map[complex128]complex128{comp1: comp2, comp2: comp1}, + Marr: map[[2]string][2]*float64{arr1: floatArr1, arr2: floatArr2}, EmptyMap: make(map[string]int), N: &[3]float64{1.5, 2.5, 3.5}, Strs: &[2]string{s1, s2}, diff --git a/libgo/go/encoding/gob/decode.go b/libgo/go/encoding/gob/decode.go index 9645dc5..8dece42 100644 --- a/libgo/go/encoding/gob/decode.go +++ b/libgo/go/encoding/gob/decode.go @@ -11,6 +11,7 @@ import ( "errors" "io" "math" + "math/bits" "reflect" ) @@ -313,12 +314,7 @@ func decUint64(i *decInstr, state *decoderState, value reflect.Value) { // (for example) transmit more compactly. This routine does the // unswizzling. func float64FromBits(u uint64) float64 { - var v uint64 - for i := 0; i < 8; i++ { - v <<= 8 - v |= u & 0xFF - u >>= 8 - } + v := bits.ReverseBytes64(u) return math.Float64frombits(v) } @@ -430,7 +426,7 @@ type decEngine struct { // decodeSingle decodes a top-level value that is not a struct and stores it in value. // Such values are preceded by a zero, making them have the memory layout of a // struct field (although with an illegal field number). -func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, value reflect.Value) { +func (dec *Decoder) decodeSingle(engine *decEngine, value reflect.Value) { state := dec.newDecoderState(&dec.buf) defer dec.freeDecoderState(state) state.fieldnum = singletonField @@ -446,7 +442,7 @@ func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, value refl // differ from ut.indir, which was computed when the engine was built. // This state cannot arise for decodeSingle, which is called directly // from the user's value, not from the innards of an engine. -func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, value reflect.Value) { +func (dec *Decoder) decodeStruct(engine *decEngine, value reflect.Value) { state := dec.newDecoderState(&dec.buf) defer dec.freeDecoderState(state) state.fieldnum = -1 @@ -538,7 +534,7 @@ func (dec *Decoder) decodeArrayHelper(state *decoderState, value reflect.Value, // decodeArray decodes an array and stores it in value. // The length is an unsigned integer preceding the elements. Even though the length is redundant // (it's part of the type), it's a useful check and is included in the encoding. -func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, value reflect.Value, elemOp decOp, length int, ovfl error, helper decHelper) { +func (dec *Decoder) decodeArray(state *decoderState, value reflect.Value, elemOp decOp, length int, ovfl error, helper decHelper) { if n := state.decodeUint(); n != uint64(length) { errorf("length mismatch in decodeArray") } @@ -546,12 +542,12 @@ func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, value re } // decodeIntoValue is a helper for map decoding. -func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Value, ovfl error) reflect.Value { - instr := &decInstr{op, 0, nil, ovfl} +func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Value, instr *decInstr) reflect.Value { v := value if isPtr { v = decAlloc(value) } + op(instr, state, v) return value } @@ -561,17 +557,24 @@ func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Va // Because the internals of maps are not visible to us, we must // use reflection rather than pointer magic. func (dec *Decoder) decodeMap(mtyp reflect.Type, state *decoderState, value reflect.Value, keyOp, elemOp decOp, ovfl error) { + n := int(state.decodeUint()) if value.IsNil() { - // Allocate map. - value.Set(reflect.MakeMap(mtyp)) + value.Set(reflect.MakeMapWithSize(mtyp, n)) } - n := int(state.decodeUint()) keyIsPtr := mtyp.Key().Kind() == reflect.Ptr elemIsPtr := mtyp.Elem().Kind() == reflect.Ptr + keyInstr := &decInstr{keyOp, 0, nil, ovfl} + elemInstr := &decInstr{elemOp, 0, nil, ovfl} + keyP := reflect.New(mtyp.Key()) + keyZ := reflect.Zero(mtyp.Key()) + elemP := reflect.New(mtyp.Elem()) + elemZ := reflect.Zero(mtyp.Elem()) for i := 0; i < n; i++ { - key := decodeIntoValue(state, keyOp, keyIsPtr, allocValue(mtyp.Key()), ovfl) - elem := decodeIntoValue(state, elemOp, elemIsPtr, allocValue(mtyp.Elem()), ovfl) + key := decodeIntoValue(state, keyOp, keyIsPtr, keyP.Elem(), keyInstr) + elem := decodeIntoValue(state, elemOp, elemIsPtr, elemP.Elem(), elemInstr) value.SetMapIndex(key, elem) + keyP.Elem().Set(keyZ) + elemP.Elem().Set(elemZ) } } @@ -657,12 +660,12 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, valu errorf("name too long (%d bytes): %.20q...", len(name), name) } // The concrete type must be registered. - registerLock.RLock() - typ, ok := nameToConcreteType[string(name)] - registerLock.RUnlock() + typi, ok := nameToConcreteType.Load(string(name)) if !ok { errorf("name not registered for interface: %q", name) } + typ := typi.(reflect.Type) + // Read the type id of the concrete value. concreteId := dec.decodeTypeSequence(true) if concreteId < 0 { @@ -813,7 +816,7 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg ovfl := overflow(name) helper := decArrayHelper[t.Elem().Kind()] op = func(i *decInstr, state *decoderState, value reflect.Value) { - state.dec.decodeArray(t, state, value, *elemOp, t.Len(), ovfl, helper) + state.dec.decodeArray(state, value, *elemOp, t.Len(), ovfl, helper) } case reflect.Map: @@ -854,7 +857,7 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg } op = func(i *decInstr, state *decoderState, value reflect.Value) { // indirect through enginePtr to delay evaluation for recursive structs. - dec.decodeStruct(*enginePtr, ut, value) + dec.decodeStruct(*enginePtr, value) } case reflect.Interface: op = func(i *decInstr, state *decoderState, value reflect.Value) { @@ -1197,9 +1200,9 @@ func (dec *Decoder) decodeValue(wireId typeId, value reflect.Value) { name := base.Name() errorf("type mismatch: no fields matched compiling decoder for %s", name) } - dec.decodeStruct(engine, ut, value) + dec.decodeStruct(engine, value) } else { - dec.decodeSingle(engine, ut, value) + dec.decodeSingle(engine, value) } } diff --git a/libgo/go/encoding/gob/decoder.go b/libgo/go/encoding/gob/decoder.go index c182941..8e0b1dd 100644 --- a/libgo/go/encoding/gob/decoder.go +++ b/libgo/go/encoding/gob/decoder.go @@ -19,6 +19,10 @@ const tooBig = 1 << 30 // A Decoder manages the receipt of type and data information read from the // remote side of a connection. +// +// The Decoder does only basic sanity checking on decoded input sizes, +// and its limits are not configurable. Take caution when decoding gob data +// from untrusted sources. type Decoder struct { mutex sync.Mutex // each item must be received atomically r io.Reader // source of the data diff --git a/libgo/go/encoding/gob/doc.go b/libgo/go/encoding/gob/doc.go index 1536574..db734ec 100644 --- a/libgo/go/encoding/gob/doc.go +++ b/libgo/go/encoding/gob/doc.go @@ -4,7 +4,7 @@ /* Package gob manages streams of gobs - binary values exchanged between an -Encoder (transmitter) and a Decoder (receiver). A typical use is transporting +Encoder (transmitter) and a Decoder (receiver). A typical use is transporting arguments and results of remote procedure calls (RPCs) such as those provided by package "net/rpc". @@ -14,28 +14,28 @@ amortizing the cost of compilation. Basics -A stream of gobs is self-describing. Each data item in the stream is preceded by +A stream of gobs is self-describing. Each data item in the stream is preceded by a specification of its type, expressed in terms of a small set of predefined -types. Pointers are not transmitted, but the things they point to are +types. Pointers are not transmitted, but the things they point to are transmitted; that is, the values are flattened. Nil pointers are not permitted, as they have no value. Recursive types work fine, but -recursive values (data with cycles) are problematic. This may change. +recursive values (data with cycles) are problematic. This may change. To use gobs, create an Encoder and present it with a series of data items as -values or addresses that can be dereferenced to values. The Encoder makes sure -all type information is sent before it is needed. At the receive side, a +values or addresses that can be dereferenced to values. The Encoder makes sure +all type information is sent before it is needed. At the receive side, a Decoder retrieves values from the encoded stream and unpacks them into local variables. Types and Values -The source and destination values/types need not correspond exactly. For structs, +The source and destination values/types need not correspond exactly. For structs, fields (identified by name) that are in the source but absent from the receiving -variable will be ignored. Fields that are in the receiving variable but missing -from the transmitted type or value will be ignored in the destination. If a field +variable will be ignored. Fields that are in the receiving variable but missing +from the transmitted type or value will be ignored in the destination. If a field with the same name is present in both, their types must be compatible. Both the receiver and transmitter will do all necessary indirection and dereferencing to -convert between gobs and actual Go values. For instance, a gob type that is +convert between gobs and actual Go values. For instance, a gob type that is schematically, struct { A, B int } @@ -63,8 +63,8 @@ Attempting to receive into these types will draw a decode error: struct { C, D int } // no field names in common Integers are transmitted two ways: arbitrary precision signed integers or -arbitrary precision unsigned integers. There is no int8, int16 etc. -discrimination in the gob format; there are only signed and unsigned integers. As +arbitrary precision unsigned integers. There is no int8, int16 etc. +discrimination in the gob format; there are only signed and unsigned integers. As described below, the transmitter sends the value in a variable-length encoding; the receiver accepts the value and stores it in the destination variable. Floating-point numbers are always sent using IEEE-754 64-bit precision (see @@ -72,7 +72,7 @@ below). Signed integers may be received into any signed integer variable: int, int16, etc.; unsigned integers may be received into any unsigned integer variable; and floating -point values may be received into any floating point variable. However, +point values may be received into any floating point variable. However, the destination variable must be able to represent the value or the decode operation will fail. @@ -106,17 +106,17 @@ Encoding Details This section documents the encoding, details that are not important for most users. Details are presented bottom-up. -An unsigned integer is sent one of two ways. If it is less than 128, it is sent -as a byte with that value. Otherwise it is sent as a minimal-length big-endian +An unsigned integer is sent one of two ways. If it is less than 128, it is sent +as a byte with that value. Otherwise it is sent as a minimal-length big-endian (high byte first) byte stream holding the value, preceded by one byte holding the -byte count, negated. Thus 0 is transmitted as (00), 7 is transmitted as (07) and +byte count, negated. Thus 0 is transmitted as (00), 7 is transmitted as (07) and 256 is transmitted as (FE 01 00). A boolean is encoded within an unsigned integer: 0 for false, 1 for true. -A signed integer, i, is encoded within an unsigned integer, u. Within u, bits 1 +A signed integer, i, is encoded within an unsigned integer, u. Within u, bits 1 upward contain the value; bit 0 says whether they should be complemented upon -receipt. The encode algorithm looks like this: +receipt. The encode algorithm looks like this: var u uint if i < 0 { @@ -127,14 +127,14 @@ receipt. The encode algorithm looks like this: encodeUnsigned(u) The low bit is therefore analogous to a sign bit, but making it the complement bit -instead guarantees that the largest negative integer is not a special case. For +instead guarantees that the largest negative integer is not a special case. For example, -129=^128=(^256>>1) encodes as (FE 01 01). Floating-point numbers are always sent as a representation of a float64 value. -That value is converted to a uint64 using math.Float64bits. The uint64 is then -byte-reversed and sent as a regular unsigned integer. The byte-reversal means the -exponent and high-precision part of the mantissa go first. Since the low bits are -often zero, this can save encoding bytes. For instance, 17.0 is encoded in only +That value is converted to a uint64 using math.Float64bits. The uint64 is then +byte-reversed and sent as a regular unsigned integer. The byte-reversal means the +exponent and high-precision part of the mantissa go first. Since the low bits are +often zero, this can save encoding bytes. For instance, 17.0 is encoded in only three bytes (FE 31 40). Strings and slices of bytes are sent as an unsigned count followed by that many @@ -151,33 +151,39 @@ is nil and not at the top level. In slices and arrays, as well as maps, all elements, even zero-valued elements, are transmitted, even if all the elements are zero. -Structs are sent as a sequence of (field number, field value) pairs. The field -value is sent using the standard gob encoding for its type, recursively. If a +Structs are sent as a sequence of (field number, field value) pairs. The field +value is sent using the standard gob encoding for its type, recursively. If a field has the zero value for its type (except for arrays; see above), it is omitted -from the transmission. The field number is defined by the type of the encoded +from the transmission. The field number is defined by the type of the encoded struct: the first field of the encoded type is field 0, the second is field 1, -etc. When encoding a value, the field numbers are delta encoded for efficiency +etc. When encoding a value, the field numbers are delta encoded for efficiency and the fields are always sent in order of increasing field number; the deltas are -therefore unsigned. The initialization for the delta encoding sets the field +therefore unsigned. The initialization for the delta encoding sets the field number to -1, so an unsigned integer field 0 with value 7 is transmitted as unsigned -delta = 1, unsigned value = 7 or (01 07). Finally, after all the fields have been -sent a terminating mark denotes the end of the struct. That mark is a delta=0 +delta = 1, unsigned value = 7 or (01 07). Finally, after all the fields have been +sent a terminating mark denotes the end of the struct. That mark is a delta=0 value, which has representation (00). Interface types are not checked for compatibility; all interface types are treated, for transmission, as members of a single "interface" type, analogous to -int or []byte - in effect they're all treated as interface{}. Interface values +int or []byte - in effect they're all treated as interface{}. Interface values are transmitted as a string identifying the concrete type being sent (a name that must be pre-defined by calling Register), followed by a byte count of the length of the following data (so the value can be skipped if it cannot be stored), followed by the usual encoding of concrete (dynamic) value stored in -the interface value. (A nil interface value is identified by the empty string +the interface value. (A nil interface value is identified by the empty string and transmits no value.) Upon receipt, the decoder verifies that the unpacked concrete item satisfies the interface of the receiving variable. -The representation of types is described below. When a type is defined on a given +If a value is passed to Encode and the type is not a struct (or pointer to struct, +etc.), for simplicity of processing it is represented as a struct of one field. +The only visible effect of this is to encode a zero byte after the value, just as +after the last field of an encoded struct, so that the decode algorithm knows when +the top-level value is complete. + +The representation of types is described below. When a type is defined on a given connection between an Encoder and Decoder, it is assigned a signed integer type -id. When Encoder.Encode(v) is called, it makes sure there is an id assigned for +id. When Encoder.Encode(v) is called, it makes sure there is an id assigned for the type of v and all its elements and then it sends the pair (typeid, encoded-v) where typeid is the type id of the encoded type of v and encoded-v is the gob encoding of the value v. @@ -223,7 +229,7 @@ If there are nested type ids, the types for all inner type ids must be defined before the top-level type id is used to describe an encoded-v. For simplicity in setup, the connection is defined to understand these types a -priori, as well as the basic gob types int, uint, etc. Their ids are: +priori, as well as the basic gob types int, uint, etc. Their ids are: bool 1 int 2 @@ -244,7 +250,7 @@ priori, as well as the basic gob types int, uint, etc. Their ids are: MapType 23 Finally, each message created by a call to Encode is preceded by an encoded -unsigned integer count of the number of bytes remaining in the message. After +unsigned integer count of the number of bytes remaining in the message. After the initial type name, interface values are wrapped the same way; in effect, the interface value acts like a recursive invocation of Encode. @@ -256,7 +262,7 @@ where * signifies zero or more repetitions and the type id of a value must be predefined or be defined before the value in the stream. Compatibility: Any future changes to the package will endeavor to maintain -compatibility with streams encoded using previous versions. That is, any released +compatibility with streams encoded using previous versions. That is, any released version of this package should be able to decode data written with any previously released version, subject to issues such as security fixes. See the Go compatibility document for background: https://golang.org/doc/go1compat @@ -315,7 +321,7 @@ StructValue: */ /* -For implementers and the curious, here is an encoded example. Given +For implementers and the curious, here is an encoded example. Given type Point struct {X, Y int} and the value p := Point{22, 33} @@ -326,14 +332,14 @@ the bytes transmitted that encode p will be: They are determined as follows. Since this is the first transmission of type Point, the type descriptor -for Point itself must be sent before the value. This is the first type +for Point itself must be sent before the value. This is the first type we've sent on this Encoder, so it has type id 65 (0 through 64 are reserved). 1f // This item (a type descriptor) is 31 bytes long. ff 81 // The negative of the id for the type we're defining, -65. // This is one byte (indicated by FF = -1) followed by - // ^-65<<1 | 1. The low 1 bit signals to complement the + // ^-65<<1 | 1. The low 1 bit signals to complement the // rest upon receipt. // Now we send a type descriptor, which is itself a struct (wireType). @@ -370,7 +376,7 @@ reserved). 00 // end of wireType.structType structure 00 // end of wireType structure -Now we can send the Point value. Again the field number resets to -1: +Now we can send the Point value. Again the field number resets to -1: 07 // this value is 7 bytes long ff 82 // the type number, 65 (1 byte (-FF) followed by 65<<1) @@ -387,7 +393,7 @@ output will be just: 07 ff 82 01 2c 01 42 00 A single non-struct value at top level is transmitted like a field with -delta tag 0. For instance, a signed integer with value 3 presented as +delta tag 0. For instance, a signed integer with value 3 presented as the argument to Encode will emit: 03 04 00 06 diff --git a/libgo/go/encoding/gob/encode.go b/libgo/go/encoding/gob/encode.go index 50cd6ad..5371e72 100644 --- a/libgo/go/encoding/gob/encode.go +++ b/libgo/go/encoding/gob/encode.go @@ -8,7 +8,9 @@ package gob import ( "encoding" + "encoding/binary" "math" + "math/bits" "reflect" "sync" ) @@ -107,14 +109,12 @@ func (state *encoderState) encodeUint(x uint64) { state.b.WriteByte(uint8(x)) return } - i := uint64Size - for x > 0 { - state.buf[i] = uint8(x) - x >>= 8 - i-- - } - state.buf[i] = uint8(i - uint64Size) // = loop count, negated - state.b.Write(state.buf[i : uint64Size+1]) + + binary.BigEndian.PutUint64(state.buf[1:], x) + bc := bits.LeadingZeros64(x) >> 3 // 8 - bytelen(x) + state.buf[bc] = uint8(bc - uint64Size) // and then we subtract 8 to get -bytelen(x) + + state.b.Write(state.buf[bc : uint64Size+1]) } // encodeInt writes an encoded signed integer to state.w. @@ -209,13 +209,7 @@ func encUint(i *encInstr, state *encoderState, v reflect.Value) { // swizzling. func floatBits(f float64) uint64 { u := math.Float64bits(f) - var v uint64 - for i := 0; i < 8; i++ { - v <<= 8 - v |= u & 0xFF - u >>= 8 - } - return v + return bits.ReverseBytes64(u) } // encFloat encodes the floating point value (float32 float64) referenced by v. @@ -404,12 +398,12 @@ func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) { } ut := userType(iv.Elem().Type()) - registerLock.RLock() - name, ok := concreteTypeToName[ut.base] - registerLock.RUnlock() + namei, ok := concreteTypeToName.Load(ut.base) if !ok { errorf("type not registered for interface: %s", ut.base) } + name := namei.(string) + // Send the name. state.encodeUint(uint64(len(name))) state.b.WriteString(name) diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index 9256848..a1ca252 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -55,6 +55,71 @@ func TestBasicEncoderDecoder(t *testing.T) { } } +func TestEncodeIntSlice(t *testing.T) { + + s8 := []int8{1, 5, 12, 22, 35, 51, 70, 92, 117} + s16 := []int16{145, 176, 210, 247, 287, 330, 376, 425, 477} + s32 := []int32{532, 590, 651, 715, 782, 852, 925, 1001, 1080} + s64 := []int64{1162, 1247, 1335, 1426, 1520, 1617, 1717, 1820, 1926} + + t.Run("int8", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s8) + + dec := NewDecoder(&sink) + res := make([]int8, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s8, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s8, res) + } + }) + + t.Run("int16", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s16) + + dec := NewDecoder(&sink) + res := make([]int16, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s16, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s16, res) + } + }) + + t.Run("int32", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s32) + + dec := NewDecoder(&sink) + res := make([]int32, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s32, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s32, res) + } + }) + + t.Run("int64", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s64) + + dec := NewDecoder(&sink) + res := make([]int64, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s64, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s64, res) + } + }) + +} + type ET0 struct { A int B string diff --git a/libgo/go/encoding/gob/error.go b/libgo/go/encoding/gob/error.go index 8b5265c..949333b 100644 --- a/libgo/go/encoding/gob/error.go +++ b/libgo/go/encoding/gob/error.go @@ -39,5 +39,4 @@ func catchError(err *error) { } *err = ge.err } - return } diff --git a/libgo/go/encoding/gob/gobencdec_test.go b/libgo/go/encoding/gob/gobencdec_test.go index ecc91ee..41a06b2 100644 --- a/libgo/go/encoding/gob/gobencdec_test.go +++ b/libgo/go/encoding/gob/gobencdec_test.go @@ -746,7 +746,7 @@ func (i *isZeroBugInterface) GobDecode(data []byte) error { } func TestGobEncodeIsZero(t *testing.T) { - x := isZeroBug{time.Now(), "hello", -55, isZeroBugArray{1, 2}, isZeroBugInterface{}} + x := isZeroBug{time.Unix(1e9, 0), "hello", -55, isZeroBugArray{1, 2}, isZeroBugInterface{}} b := new(bytes.Buffer) enc := NewEncoder(b) err := enc.Encode(x) diff --git a/libgo/go/encoding/gob/timing_test.go b/libgo/go/encoding/gob/timing_test.go index 424b7e6..3478bd2 100644 --- a/libgo/go/encoding/gob/timing_test.go +++ b/libgo/go/encoding/gob/timing_test.go @@ -8,6 +8,7 @@ import ( "bytes" "io" "os" + "reflect" "runtime" "testing" ) @@ -132,89 +133,60 @@ func TestCountDecodeMallocs(t *testing.T) { } } +func benchmarkEncodeSlice(b *testing.B, a interface{}) { + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + for pb.Next() { + buf.Reset() + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + } + }) +} + func BenchmarkEncodeComplex128Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]complex128, 1000) for i := range a { a[i] = 1.2 + 3.4i } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeFloat64Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]float64, 1000) for i := range a { a[i] = 1.23e4 } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeInt32Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]int32, 1000) for i := range a { - a[i] = 1234 - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } + a[i] = int32(i * 100) } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeStringSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]string, 1000) for i := range a { a[i] = "now is the time" } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeInterfaceSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]interface{}, 1000) for i := range a { a[i] = "now is the time" } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } // benchmarkBuf is a read buffer we can reset @@ -245,120 +217,96 @@ func (b *benchmarkBuf) reset() { b.offset = 0 } -func BenchmarkDecodeComplex128Slice(b *testing.B) { +func benchmarkDecodeSlice(b *testing.B, a interface{}) { var buf bytes.Buffer enc := NewEncoder(&buf) - a := make([]complex128, 1000) - for i := range a { - a[i] = 1.2 + 3.4i - } err := enc.Encode(a) if err != nil { b.Fatal(err) } - x := make([]complex128, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} + + ra := reflect.ValueOf(a) + rt := ra.Type() b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) + + b.RunParallel(func(pb *testing.PB) { + // TODO(#19025): Move per-thread allocation before ResetTimer. + rp := reflect.New(rt) + rp.Elem().Set(reflect.MakeSlice(rt, ra.Len(), ra.Cap())) + p := rp.Interface() + + bbuf := benchmarkBuf{data: buf.Bytes()} + + for pb.Next() { + bbuf.reset() + dec := NewDecoder(&bbuf) + err := dec.Decode(p) + if err != nil { + b.Fatal(err) + } } + }) +} + +func BenchmarkDecodeComplex128Slice(b *testing.B) { + a := make([]complex128, 1000) + for i := range a { + a[i] = 1.2 + 3.4i } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeFloat64Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]float64, 1000) for i := range a { a[i] = 1.23e4 } - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - x := make([]float64, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} - b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) - } - } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeInt32Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]int32, 1000) for i := range a { a[i] = 1234 } - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - x := make([]int32, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} - b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) - } - } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeStringSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]string, 1000) for i := range a { a[i] = "now is the time" } - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - x := make([]string, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} - b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) - } - } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeInterfaceSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]interface{}, 1000) for i := range a { a[i] = "now is the time" } - err := enc.Encode(a) + benchmarkDecodeSlice(b, a) +} + +func BenchmarkDecodeMap(b *testing.B) { + count := 1000 + m := make(map[int]int, count) + for i := 0; i < count; i++ { + m[i] = i + } + var buf bytes.Buffer + enc := NewEncoder(&buf) + err := enc.Encode(m) if err != nil { b.Fatal(err) } - x := make([]interface{}, 1000) bbuf := benchmarkBuf{data: buf.Bytes()} b.ResetTimer() for i := 0; i < b.N; i++ { + var rm map[int]int bbuf.reset() dec := NewDecoder(&bbuf) - err := dec.Decode(&x) + err := dec.Decode(&rm) if err != nil { b.Fatal(i, err) } diff --git a/libgo/go/encoding/gob/type.go b/libgo/go/encoding/gob/type.go index c27f7e9..31c0ef7 100644 --- a/libgo/go/encoding/gob/type.go +++ b/libgo/go/encoding/gob/type.go @@ -36,31 +36,21 @@ const ( xText // encoding.TextMarshaler or encoding.TextUnmarshaler ) -var ( - // Protected by an RWMutex because we read it a lot and write - // it only when we see a new type, typically when compiling. - userTypeLock sync.RWMutex - userTypeCache = make(map[reflect.Type]*userTypeInfo) -) +var userTypeCache sync.Map // map[reflect.Type]*userTypeInfo // validType returns, and saves, the information associated with user-provided type rt. // If the user type is not valid, err will be non-nil. To be used when the error handler // is not set up. -func validUserType(rt reflect.Type) (ut *userTypeInfo, err error) { - userTypeLock.RLock() - ut = userTypeCache[rt] - userTypeLock.RUnlock() - if ut != nil { - return - } - // Now set the value under the write lock. - userTypeLock.Lock() - defer userTypeLock.Unlock() - if ut = userTypeCache[rt]; ut != nil { - // Lost the race; not a problem. - return +func validUserType(rt reflect.Type) (*userTypeInfo, error) { + if ui, ok := userTypeCache.Load(rt); ok { + return ui.(*userTypeInfo), nil } - ut = new(userTypeInfo) + + // Construct a new userTypeInfo and atomically add it to the userTypeCache. + // If we lose the race, we'll waste a little CPU and create a little garbage + // but return the existing value anyway. + + ut := new(userTypeInfo) ut.base = rt ut.user = rt // A type that is just a cycle of pointers (such as type T *T) cannot @@ -108,8 +98,8 @@ func validUserType(rt reflect.Type) (ut *userTypeInfo, err error) { // ut.externalDec, ut.decIndir = xText, indir // } - userTypeCache[rt] = ut - return + ui, _ := userTypeCache.LoadOrStore(rt, ut) + return ui.(*userTypeInfo), nil } var ( @@ -808,9 +798,8 @@ type GobDecoder interface { } var ( - registerLock sync.RWMutex - nameToConcreteType = make(map[string]reflect.Type) - concreteTypeToName = make(map[reflect.Type]string) + nameToConcreteType sync.Map // map[string]reflect.Type + concreteTypeToName sync.Map // map[reflect.Type]string ) // RegisterName is like Register but uses the provided name rather than the @@ -820,21 +809,22 @@ func RegisterName(name string, value interface{}) { // reserved for nil panic("attempt to register empty name") } - registerLock.Lock() - defer registerLock.Unlock() + ut := userType(reflect.TypeOf(value)) + // Check for incompatible duplicates. The name must refer to the // same user type, and vice versa. - if t, ok := nameToConcreteType[name]; ok && t != ut.user { + + // Store the name and type provided by the user.... + if t, dup := nameToConcreteType.LoadOrStore(name, reflect.TypeOf(value)); dup && t != ut.user { panic(fmt.Sprintf("gob: registering duplicate types for %q: %s != %s", name, t, ut.user)) } - if n, ok := concreteTypeToName[ut.base]; ok && n != name { + + // but the flattened type in the type table, since that's what decode needs. + if n, dup := concreteTypeToName.LoadOrStore(ut.base, name); dup && n != name { + nameToConcreteType.Delete(name) panic(fmt.Sprintf("gob: registering duplicate names for %s: %q != %q", ut.user, n, name)) } - // Store the name and type provided by the user.... - nameToConcreteType[name] = reflect.TypeOf(value) - // but the flattened type in the type table, since that's what decode needs. - concreteTypeToName[ut.base] = name } // Register records a type, identified by a value for that type, under its diff --git a/libgo/go/encoding/gob/type_test.go b/libgo/go/encoding/gob/type_test.go index e230d22..14f25d8 100644 --- a/libgo/go/encoding/gob/type_test.go +++ b/libgo/go/encoding/gob/type_test.go @@ -178,9 +178,7 @@ func TestRegistrationNaming(t *testing.T) { Register(tc.t) tct := reflect.TypeOf(tc.t) - registerLock.RLock() - ct := nameToConcreteType[tc.name] - registerLock.RUnlock() + ct, _ := nameToConcreteType.Load(tc.name) if ct != tct { t.Errorf("nameToConcreteType[%q] = %v, want %v", tc.name, ct, tct) } @@ -188,7 +186,7 @@ func TestRegistrationNaming(t *testing.T) { if tct.Kind() == reflect.Ptr { tct = tct.Elem() } - if n := concreteTypeToName[tct]; n != tc.name { + if n, _ := concreteTypeToName.Load(tct); n != tc.name { t.Errorf("concreteTypeToName[%v] got %v, want %v", tct, n, tc.name) } } diff --git a/libgo/go/encoding/hex/hex.go b/libgo/go/encoding/hex/hex.go index b43c1c4..2768f1b 100644 --- a/libgo/go/encoding/hex/hex.go +++ b/libgo/go/encoding/hex/hex.go @@ -12,10 +12,7 @@ import ( "io" ) -var hextable = [16]byte{ - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'a', 'b', 'c', 'd', 'e', 'f', -} +const hextable = "0123456789abcdef" // EncodedLen returns the length of an encoding of n source bytes. // Specifically, it returns n * 2. diff --git a/libgo/go/encoding/json/bench_test.go b/libgo/go/encoding/json/bench_test.go index cd7380b..85d7ae0 100644 --- a/libgo/go/encoding/json/bench_test.go +++ b/libgo/go/encoding/json/bench_test.go @@ -82,12 +82,14 @@ func BenchmarkCodeEncoder(b *testing.B) { codeInit() b.StartTimer() } - enc := NewEncoder(ioutil.Discard) - for i := 0; i < b.N; i++ { - if err := enc.Encode(&codeStruct); err != nil { - b.Fatal("Encode:", err) + b.RunParallel(func(pb *testing.PB) { + enc := NewEncoder(ioutil.Discard) + for pb.Next() { + if err := enc.Encode(&codeStruct); err != nil { + b.Fatal("Encode:", err) + } } - } + }) b.SetBytes(int64(len(codeJSON))) } @@ -97,11 +99,13 @@ func BenchmarkCodeMarshal(b *testing.B) { codeInit() b.StartTimer() } - for i := 0; i < b.N; i++ { - if _, err := Marshal(&codeStruct); err != nil { - b.Fatal("Marshal:", err) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(&codeStruct); err != nil { + b.Fatal("Marshal:", err) + } } - } + }) b.SetBytes(int64(len(codeJSON))) } @@ -111,19 +115,21 @@ func BenchmarkCodeDecoder(b *testing.B) { codeInit() b.StartTimer() } - var buf bytes.Buffer - dec := NewDecoder(&buf) - var r codeResponse - for i := 0; i < b.N; i++ { - buf.Write(codeJSON) - // hide EOF - buf.WriteByte('\n') - buf.WriteByte('\n') - buf.WriteByte('\n') - if err := dec.Decode(&r); err != nil { - b.Fatal("Decode:", err) + b.RunParallel(func(pb *testing.PB) { + var buf bytes.Buffer + dec := NewDecoder(&buf) + var r codeResponse + for pb.Next() { + buf.Write(codeJSON) + // hide EOF + buf.WriteByte('\n') + buf.WriteByte('\n') + buf.WriteByte('\n') + if err := dec.Decode(&r); err != nil { + b.Fatal("Decode:", err) + } } - } + }) b.SetBytes(int64(len(codeJSON))) } @@ -155,12 +161,14 @@ func BenchmarkCodeUnmarshal(b *testing.B) { codeInit() b.StartTimer() } - for i := 0; i < b.N; i++ { - var r codeResponse - if err := Unmarshal(codeJSON, &r); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + var r codeResponse + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) b.SetBytes(int64(len(codeJSON))) } @@ -170,54 +178,75 @@ func BenchmarkCodeUnmarshalReuse(b *testing.B) { codeInit() b.StartTimer() } - var r codeResponse - for i := 0; i < b.N; i++ { - if err := Unmarshal(codeJSON, &r); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + var r codeResponse + for pb.Next() { + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) + // TODO(bcmills): Is there a missing b.SetBytes here? } func BenchmarkUnmarshalString(b *testing.B) { data := []byte(`"hello, world"`) - var s string - - for i := 0; i < b.N; i++ { - if err := Unmarshal(data, &s); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + var s string + for pb.Next() { + if err := Unmarshal(data, &s); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) } func BenchmarkUnmarshalFloat64(b *testing.B) { - var f float64 data := []byte(`3.14`) - - for i := 0; i < b.N; i++ { - if err := Unmarshal(data, &f); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + var f float64 + for pb.Next() { + if err := Unmarshal(data, &f); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) } func BenchmarkUnmarshalInt64(b *testing.B) { - var x int64 data := []byte(`3`) - - for i := 0; i < b.N; i++ { - if err := Unmarshal(data, &x); err != nil { - b.Fatal("Unmarshal:", err) + b.RunParallel(func(pb *testing.PB) { + var x int64 + for pb.Next() { + if err := Unmarshal(data, &x); err != nil { + b.Fatal("Unmarshal:", err) + } } - } + }) } func BenchmarkIssue10335(b *testing.B) { b.ReportAllocs() - var s struct{} j := []byte(`{"a":{ }}`) - for n := 0; n < b.N; n++ { - if err := Unmarshal(j, &s); err != nil { - b.Fatal(err) + b.RunParallel(func(pb *testing.PB) { + var s struct{} + for pb.Next() { + if err := Unmarshal(j, &s); err != nil { + b.Fatal(err) + } } - } + }) +} + +func BenchmarkUnmapped(b *testing.B) { + b.ReportAllocs() + j := []byte(`{"s": "hello", "y": 2, "o": {"x": 0}, "a": [1, 99, {"x": 1}]}`) + b.RunParallel(func(pb *testing.PB) { + var s struct{} + for pb.Next() { + if err := Unmarshal(j, &s); err != nil { + b.Fatal(err) + } + } + }) } diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index 77fc460..420a07e 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -22,7 +22,8 @@ import ( ) // Unmarshal parses the JSON-encoded data and stores the result -// in the value pointed to by v. +// in the value pointed to by v. If v is nil or not a pointer, +// Unmarshal returns an InvalidUnmarshalError. // // Unmarshal uses the inverse of the encodings that // Marshal uses, allocating maps, slices, and pointers as necessary, @@ -78,7 +79,9 @@ import ( // or if a JSON number overflows the target type, Unmarshal // skips that field and completes the unmarshaling as best it can. // If no more serious errors are encountered, Unmarshal returns -// an UnmarshalTypeError describing the earliest such error. +// an UnmarshalTypeError describing the earliest such error. In any +// case, it's not guaranteed that all the remaining fields following +// the problematic one will be unmarshaled into the target object. // // The JSON null value unmarshals into an interface, map, pointer, or slice // by setting that Go value to nil. Because null is often used in JSON to mean diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index 8f21dda..0371f0a 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -332,10 +332,7 @@ type encOpts struct { type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts) -var encoderCache struct { - sync.RWMutex - m map[reflect.Type]encoderFunc -} +var encoderCache sync.Map // map[reflect.Type]encoderFunc func valueEncoder(v reflect.Value) encoderFunc { if !v.IsValid() { @@ -345,36 +342,31 @@ func valueEncoder(v reflect.Value) encoderFunc { } func typeEncoder(t reflect.Type) encoderFunc { - encoderCache.RLock() - f := encoderCache.m[t] - encoderCache.RUnlock() - if f != nil { - return f + if fi, ok := encoderCache.Load(t); ok { + return fi.(encoderFunc) } // To deal with recursive types, populate the map with an // indirect func before we build it. This type waits on the // real func (f) to be ready and then calls it. This indirect // func is only used for recursive types. - encoderCache.Lock() - if encoderCache.m == nil { - encoderCache.m = make(map[reflect.Type]encoderFunc) - } - var wg sync.WaitGroup + var ( + wg sync.WaitGroup + f encoderFunc + ) wg.Add(1) - encoderCache.m[t] = func(e *encodeState, v reflect.Value, opts encOpts) { + fi, loaded := encoderCache.LoadOrStore(t, encoderFunc(func(e *encodeState, v reflect.Value, opts encOpts) { wg.Wait() f(e, v, opts) + })) + if loaded { + return fi.(encoderFunc) } - encoderCache.Unlock() - // Compute fields without lock. - // Might duplicate effort but won't hold other computations back. + // Compute the real encoder and replace the indirect func with it. f = newTypeEncoder(t, true) wg.Done() - encoderCache.Lock() - encoderCache.m[t] = f - encoderCache.Unlock() + encoderCache.Store(t, f) return f } @@ -1101,7 +1093,22 @@ func typeFields(t reflect.Type) []field { // Scan f.typ for fields to include. for i := 0; i < f.typ.NumField(); i++ { sf := f.typ.Field(i) - if sf.PkgPath != "" && !sf.Anonymous { // unexported + if sf.Anonymous { + t := sf.Type + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + // If embedded, StructField.PkgPath is not a reliable + // indicator of whether the field is exported. + // See https://golang.org/issue/21122 + if !isExported(t.Name()) && t.Kind() != reflect.Struct { + // Ignore embedded fields of unexported non-struct types. + // Do not ignore embedded fields of unexported struct types + // since they may have exported fields. + continue + } + } else if sf.PkgPath != "" { + // Ignore unexported non-embedded fields. continue } tag := sf.Tag.Get("json") @@ -1219,6 +1226,12 @@ func typeFields(t reflect.Type) []field { return fields } +// isExported reports whether the identifier is exported. +func isExported(id string) bool { + r, _ := utf8.DecodeRuneInString(id) + return unicode.IsUpper(r) +} + // dominantField looks through the fields, all of which are known to // have the same name, to find the single field that dominates the // others using Go's embedding rules, modified by the presence of diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index 6d574cf..3fda6a0 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -253,23 +253,167 @@ func TestMarshalerEscaping(t *testing.T) { } } -type IntType int - -type MyStruct struct { - IntType -} - -func TestAnonymousNonstruct(t *testing.T) { - var i IntType = 11 - a := MyStruct{i} - const want = `{"IntType":11}` +func TestAnonymousFields(t *testing.T) { + tests := []struct { + label string // Test name + makeInput func() interface{} // Function to create input value + want string // Expected JSON output + }{{ + // Both S1 and S2 have a field named X. From the perspective of S, + // it is ambiguous which one X refers to. + // This should not serialize either field. + label: "AmbiguousField", + makeInput: func() interface{} { + type ( + S1 struct{ x, X int } + S2 struct{ x, X int } + S struct { + S1 + S2 + } + ) + return S{S1{1, 2}, S2{3, 4}} + }, + want: `{}`, + }, { + label: "DominantField", + // Both S1 and S2 have a field named X, but since S has an X field as + // well, it takes precedence over S1.X and S2.X. + makeInput: func() interface{} { + type ( + S1 struct{ x, X int } + S2 struct{ x, X int } + S struct { + S1 + S2 + x, X int + } + ) + return S{S1{1, 2}, S2{3, 4}, 5, 6} + }, + want: `{"X":6}`, + }, { + // Unexported embedded field of non-struct type should not be serialized. + label: "UnexportedEmbeddedInt", + makeInput: func() interface{} { + type ( + myInt int + S struct{ myInt } + ) + return S{5} + }, + want: `{}`, + }, { + // Exported embedded field of non-struct type should be serialized. + label: "ExportedEmbeddedInt", + makeInput: func() interface{} { + type ( + MyInt int + S struct{ MyInt } + ) + return S{5} + }, + want: `{"MyInt":5}`, + }, { + // Unexported embedded field of pointer to non-struct type + // should not be serialized. + label: "UnexportedEmbeddedIntPointer", + makeInput: func() interface{} { + type ( + myInt int + S struct{ *myInt } + ) + s := S{new(myInt)} + *s.myInt = 5 + return s + }, + want: `{}`, + }, { + // Exported embedded field of pointer to non-struct type + // should be serialized. + label: "ExportedEmbeddedIntPointer", + makeInput: func() interface{} { + type ( + MyInt int + S struct{ *MyInt } + ) + s := S{new(MyInt)} + *s.MyInt = 5 + return s + }, + want: `{"MyInt":5}`, + }, { + // Exported fields of embedded structs should have their + // exported fields be serialized regardless of whether the struct types + // themselves are exported. + label: "EmbeddedStruct", + makeInput: func() interface{} { + type ( + s1 struct{ x, X int } + S2 struct{ y, Y int } + S struct { + s1 + S2 + } + ) + return S{s1{1, 2}, S2{3, 4}} + }, + want: `{"X":2,"Y":4}`, + }, { + // Exported fields of pointers to embedded structs should have their + // exported fields be serialized regardless of whether the struct types + // themselves are exported. + label: "EmbeddedStructPointer", + makeInput: func() interface{} { + type ( + s1 struct{ x, X int } + S2 struct{ y, Y int } + S struct { + *s1 + *S2 + } + ) + return S{&s1{1, 2}, &S2{3, 4}} + }, + want: `{"X":2,"Y":4}`, + }, { + // Exported fields on embedded unexported structs at multiple levels + // of nesting should still be serialized. + label: "NestedStructAndInts", + makeInput: func() interface{} { + type ( + MyInt1 int + MyInt2 int + myInt int + s2 struct { + MyInt2 + myInt + } + s1 struct { + MyInt1 + myInt + s2 + } + S struct { + s1 + myInt + } + ) + return S{s1{1, 2, s2{3, 4}}, 6} + }, + want: `{"MyInt1":1,"MyInt2":3}`, + }} - b, err := Marshal(a) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if got := string(b); got != want { - t.Errorf("got %q, want %q", got, want) + for _, tt := range tests { + t.Run(tt.label, func(t *testing.T) { + b, err := Marshal(tt.makeInput()) + if err != nil { + t.Fatalf("Marshal() = %v, want nil error", err) + } + if string(b) != tt.want { + t.Fatalf("Marshal() = %q, want %q", b, tt.want) + } + }) } } diff --git a/libgo/go/encoding/json/scanner.go b/libgo/go/encoding/json/scanner.go index a6d8706..ae34418 100644 --- a/libgo/go/encoding/json/scanner.go +++ b/libgo/go/encoding/json/scanner.go @@ -15,6 +15,11 @@ package json import "strconv" +// Valid reports whether data is a valid JSON encoding. +func Valid(data []byte) bool { + return checkValid(data, &scanner{}) == nil +} + // checkValid verifies that data is valid JSON-encoded data. // scan is passed in for use by checkValid to avoid an allocation. func checkValid(data []byte, scan *scanner) error { diff --git a/libgo/go/encoding/json/scanner_test.go b/libgo/go/encoding/json/scanner_test.go index c5c1be3..0d4518a 100644 --- a/libgo/go/encoding/json/scanner_test.go +++ b/libgo/go/encoding/json/scanner_test.go @@ -12,6 +12,26 @@ import ( "testing" ) +var validTests = []struct { + data string + ok bool +}{ + {`foo`, false}, + {`}{`, false}, + {`{]`, false}, + {`{}`, true}, + {`{"foo":"bar"}`, true}, + {`{"foo":"bar","bar":{"baz":["qux"]}}`, true}, +} + +func TestValid(t *testing.T) { + for _, tt := range validTests { + if ok := Valid([]byte(tt.data)); ok != tt.ok { + t.Errorf("Valid(%#q) = %v, want %v", tt.data, ok, tt.ok) + } + } +} + // Tests of simple examples. type example struct { diff --git a/libgo/go/encoding/json/stream_test.go b/libgo/go/encoding/json/stream_test.go index 84edeb1..d0b3ffb 100644 --- a/libgo/go/encoding/json/stream_test.go +++ b/libgo/go/encoding/json/stream_test.go @@ -268,11 +268,13 @@ func BenchmarkEncoderEncode(b *testing.B) { X, Y string } v := &T{"foo", "bar"} - for i := 0; i < b.N; i++ { - if err := NewEncoder(ioutil.Discard).Encode(v); err != nil { - b.Fatal(err) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := NewEncoder(ioutil.Discard).Encode(v); err != nil { + b.Fatal(err) + } } - } + }) } type tokenStreamCase struct { diff --git a/libgo/go/encoding/pem/pem.go b/libgo/go/encoding/pem/pem.go index fbf4999..5e1ab90 100644 --- a/libgo/go/encoding/pem/pem.go +++ b/libgo/go/encoding/pem/pem.go @@ -135,20 +135,26 @@ func Decode(data []byte) (p *Block, rest []byte) { return decodeError(data, rest) } - // After the "-----" of the ending line should be the same type and a - // final five dashes. + // After the "-----" of the ending line, there should be the same type + // and then a final five dashes. endTrailer := rest[endTrailerIndex:] endTrailerLen := len(typeLine) + len(pemEndOfLine) if len(endTrailer) < endTrailerLen { return decodeError(data, rest) } + restOfEndLine := endTrailer[endTrailerLen:] endTrailer = endTrailer[:endTrailerLen] if !bytes.HasPrefix(endTrailer, typeLine) || !bytes.HasSuffix(endTrailer, pemEndOfLine) { return decodeError(data, rest) } + // The line must end with only whitespace. + if s, _ := getLine(restOfEndLine); len(s) != 0 { + return decodeError(data, rest) + } + base64Data := removeWhitespace(rest[:endIndex]) p.Bytes = make([]byte, base64.StdEncoding.DecodedLen(len(base64Data))) n, err := base64.StdEncoding.Decode(p.Bytes, base64Data) diff --git a/libgo/go/encoding/pem/pem_test.go b/libgo/go/encoding/pem/pem_test.go index 6321dec..1a1250a 100644 --- a/libgo/go/encoding/pem/pem_test.go +++ b/libgo/go/encoding/pem/pem_test.go @@ -83,6 +83,16 @@ const pemTooFewEndingDashes = ` dGVzdA== -----END FOO----` +const pemTooManyEndingDashes = ` +-----BEGIN FOO----- +dGVzdA== +-----END FOO------` + +const pemTrailingNonWhitespace = ` +-----BEGIN FOO----- +dGVzdA== +-----END FOO----- .` + const pemWrongEndingType = ` -----BEGIN FOO----- dGVzdA== @@ -102,6 +112,14 @@ var badPEMTests = []struct { pemTooFewEndingDashes, }, { + "too many trailing dashes", + pemTooManyEndingDashes, + }, + { + "trailing non-whitespace", + pemTrailingNonWhitespace, + }, + { "incorrect ending type", pemWrongEndingType, }, @@ -188,10 +206,20 @@ func TestLineBreaker(t *testing.T) { } func TestFuzz(t *testing.T) { + // PEM is a text-based format. Assume header fields with leading/trailing spaces + // or embedded newlines will not round trip correctly and don't need to be tested. + isBad := func(s string) bool { + return strings.ContainsAny(s, "\r\n") || strings.TrimSpace(s) != s + } + testRoundtrip := func(block Block) bool { - for key := range block.Headers { - if strings.Contains(key, ":") { - // Keys with colons cannot be encoded. + if isBad(block.Type) { + return true + } + for key, val := range block.Headers { + // Reject bad key/val. + // Also, keys with colons cannot be encoded, because : is the key: val separator. + if isBad(key) || isBad(val) || strings.Contains(key, ":") { return true } } diff --git a/libgo/go/encoding/xml/marshal_test.go b/libgo/go/encoding/xml/marshal_test.go index 0126146..674c6b5 100644 --- a/libgo/go/encoding/xml/marshal_test.go +++ b/libgo/go/encoding/xml/marshal_test.go @@ -1652,28 +1652,31 @@ func TestMarshal(t *testing.T) { if test.UnmarshalOnly { continue } - data, err := Marshal(test.Value) - if err != nil { - if test.MarshalError == "" { - t.Errorf("#%d: marshal(%#v): %s", idx, test.Value, err) - continue + + t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { + data, err := Marshal(test.Value) + if err != nil { + if test.MarshalError == "" { + t.Errorf("marshal(%#v): %s", test.Value, err) + return + } + if !strings.Contains(err.Error(), test.MarshalError) { + t.Errorf("marshal(%#v): %s, want %q", test.Value, err, test.MarshalError) + } + return } - if !strings.Contains(err.Error(), test.MarshalError) { - t.Errorf("#%d: marshal(%#v): %s, want %q", idx, test.Value, err, test.MarshalError) + if test.MarshalError != "" { + t.Errorf("Marshal succeeded, want error %q", test.MarshalError) + return } - continue - } - if test.MarshalError != "" { - t.Errorf("#%d: Marshal succeeded, want error %q", idx, test.MarshalError) - continue - } - if got, want := string(data), test.ExpectXML; got != want { - if strings.Contains(want, "\n") { - t.Errorf("#%d: marshal(%#v):\nHAVE:\n%s\nWANT:\n%s", idx, test.Value, got, want) - } else { - t.Errorf("#%d: marshal(%#v):\nhave %#q\nwant %#q", idx, test.Value, got, want) + if got, want := string(data), test.ExpectXML; got != want { + if strings.Contains(want, "\n") { + t.Errorf("marshal(%#v):\nHAVE:\n%s\nWANT:\n%s", test.Value, got, want) + } else { + t.Errorf("marshal(%#v):\nhave %#q\nwant %#q", test.Value, got, want) + } } - } + }) } } @@ -1781,27 +1784,29 @@ func TestUnmarshal(t *testing.T) { dest := reflect.New(vt.Elem()).Interface() err := Unmarshal([]byte(test.ExpectXML), dest) - switch fix := dest.(type) { - case *Feed: - fix.Author.InnerXML = "" - for i := range fix.Entry { - fix.Entry[i].Author.InnerXML = "" + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + switch fix := dest.(type) { + case *Feed: + fix.Author.InnerXML = "" + for i := range fix.Entry { + fix.Entry[i].Author.InnerXML = "" + } } - } - if err != nil { - if test.UnmarshalError == "" { - t.Errorf("#%d: unmarshal(%#v): %s", i, test.ExpectXML, err) - continue + if err != nil { + if test.UnmarshalError == "" { + t.Errorf("unmarshal(%#v): %s", test.ExpectXML, err) + return + } + if !strings.Contains(err.Error(), test.UnmarshalError) { + t.Errorf("unmarshal(%#v): %s, want %q", test.ExpectXML, err, test.UnmarshalError) + } + return } - if !strings.Contains(err.Error(), test.UnmarshalError) { - t.Errorf("#%d: unmarshal(%#v): %s, want %q", i, test.ExpectXML, err, test.UnmarshalError) + if got, want := dest, test.Value; !reflect.DeepEqual(got, want) { + t.Errorf("unmarshal(%q):\nhave %#v\nwant %#v", test.ExpectXML, got, want) } - continue - } - if got, want := dest, test.Value; !reflect.DeepEqual(got, want) { - t.Errorf("#%d: unmarshal(%q):\nhave %#v\nwant %#v", i, test.ExpectXML, got, want) - } + }) } } @@ -1896,17 +1901,21 @@ func TestMarshalFlush(t *testing.T) { func BenchmarkMarshal(b *testing.B) { b.ReportAllocs() - for i := 0; i < b.N; i++ { - Marshal(atomValue) - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + Marshal(atomValue) + } + }) } func BenchmarkUnmarshal(b *testing.B) { b.ReportAllocs() xml := []byte(atomXml) - for i := 0; i < b.N; i++ { - Unmarshal(xml, &Feed{}) - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + Unmarshal(xml, &Feed{}) + } + }) } // golang.org/issue/6556 @@ -2428,10 +2437,7 @@ func TestIssue16158(t *testing.T) { err := Unmarshal([]byte(data), &struct { B byte `xml:"b,attr,omitempty"` }{}) - - // For Go 1.8.1 we've restored the old "no errors reported" behavior. - // We'll try again in Go 1.9 to report errors. - if err != nil { - t.Errorf("Unmarshal: expected nil, got error") + if err == nil { + t.Errorf("Unmarshal: expected error, got nil") } } diff --git a/libgo/go/encoding/xml/read.go b/libgo/go/encoding/xml/read.go index 799b57e..000d9fb 100644 --- a/libgo/go/encoding/xml/read.go +++ b/libgo/go/encoding/xml/read.go @@ -120,6 +120,9 @@ import ( // Unmarshal maps an XML element to a pointer by setting the pointer // to a freshly allocated value and then mapping the element to that value. // +// A missing element or empty attribute value will be unmarshaled as a zero value. +// If the field is a slice, a zero value will be appended to the field. Otherwise, the +// field will be set to its zero value. func Unmarshal(data []byte, v interface{}) error { return NewDecoder(bytes.NewReader(data)).Decode(v) } @@ -211,7 +214,7 @@ func (p *Decoder) unmarshalInterface(val Unmarshaler, start *StartElement) error // unmarshalTextInterface unmarshals a single XML element into val. // The chardata contained in the element (but not its children) // is passed to the text unmarshaler. -func (p *Decoder) unmarshalTextInterface(val encoding.TextUnmarshaler, start *StartElement) error { +func (p *Decoder) unmarshalTextInterface(val encoding.TextUnmarshaler) error { var buf []byte depth := 1 for depth > 0 { @@ -285,8 +288,7 @@ func (p *Decoder) unmarshalAttr(val reflect.Value, attr Attr) error { return nil } - copyValue(val, []byte(attr.Value)) - return nil + return copyValue(val, []byte(attr.Value)) } var ( @@ -342,13 +344,13 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { } if val.CanInterface() && val.Type().Implements(textUnmarshalerType) { - return p.unmarshalTextInterface(val.Interface().(encoding.TextUnmarshaler), start) + return p.unmarshalTextInterface(val.Interface().(encoding.TextUnmarshaler)) } if val.CanAddr() { pv := val.Addr() if pv.CanInterface() && pv.Type().Implements(textUnmarshalerType) { - return p.unmarshalTextInterface(pv.Interface().(encoding.TextUnmarshaler), start) + return p.unmarshalTextInterface(pv.Interface().(encoding.TextUnmarshaler)) } } @@ -608,24 +610,40 @@ func copyValue(dst reflect.Value, src []byte) (err error) { default: return errors.New("cannot unmarshal into " + dst0.Type().String()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if len(src) == 0 { + dst.SetInt(0) + return nil + } itmp, err := strconv.ParseInt(string(src), 10, dst.Type().Bits()) if err != nil { return err } dst.SetInt(itmp) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + if len(src) == 0 { + dst.SetUint(0) + return nil + } utmp, err := strconv.ParseUint(string(src), 10, dst.Type().Bits()) if err != nil { return err } dst.SetUint(utmp) case reflect.Float32, reflect.Float64: + if len(src) == 0 { + dst.SetFloat(0) + return nil + } ftmp, err := strconv.ParseFloat(string(src), dst.Type().Bits()) if err != nil { return err } dst.SetFloat(ftmp) case reflect.Bool: + if len(src) == 0 { + dst.SetBool(false) + return nil + } value, err := strconv.ParseBool(strings.TrimSpace(string(src))) if err != nil { return err diff --git a/libgo/go/encoding/xml/read_test.go b/libgo/go/encoding/xml/read_test.go index 273c303..a1eb516 100644 --- a/libgo/go/encoding/xml/read_test.go +++ b/libgo/go/encoding/xml/read_test.go @@ -752,3 +752,159 @@ func TestInvalidInnerXMLType(t *testing.T) { t.Errorf("NotInnerXML = %v, want nil", v.NotInnerXML) } } + +type Child struct { + G struct { + I int + } +} + +type ChildToEmbed struct { + X bool +} + +type Parent struct { + I int + IPtr *int + Is []int + IPtrs []*int + F float32 + FPtr *float32 + Fs []float32 + FPtrs []*float32 + B bool + BPtr *bool + Bs []bool + BPtrs []*bool + Bytes []byte + BytesPtr *[]byte + S string + SPtr *string + Ss []string + SPtrs []*string + MyI MyInt + Child Child + Children []Child + ChildPtr *Child + ChildToEmbed +} + +const ( + emptyXML = ` +<Parent> + <I></I> + <IPtr></IPtr> + <Is></Is> + <IPtrs></IPtrs> + <F></F> + <FPtr></FPtr> + <Fs></Fs> + <FPtrs></FPtrs> + <B></B> + <BPtr></BPtr> + <Bs></Bs> + <BPtrs></BPtrs> + <Bytes></Bytes> + <BytesPtr></BytesPtr> + <S></S> + <SPtr></SPtr> + <Ss></Ss> + <SPtrs></SPtrs> + <MyI></MyI> + <Child></Child> + <Children></Children> + <ChildPtr></ChildPtr> + <X></X> +</Parent> +` +) + +// github.com/golang/go/issues/13417 +func TestUnmarshalEmptyValues(t *testing.T) { + // Test first with a zero-valued dst. + v := new(Parent) + if err := Unmarshal([]byte(emptyXML), v); err != nil { + t.Fatalf("zero: Unmarshal failed: got %v", err) + } + + zBytes, zInt, zStr, zFloat, zBool := []byte{}, 0, "", float32(0), false + want := &Parent{ + IPtr: &zInt, + Is: []int{zInt}, + IPtrs: []*int{&zInt}, + FPtr: &zFloat, + Fs: []float32{zFloat}, + FPtrs: []*float32{&zFloat}, + BPtr: &zBool, + Bs: []bool{zBool}, + BPtrs: []*bool{&zBool}, + Bytes: []byte{}, + BytesPtr: &zBytes, + SPtr: &zStr, + Ss: []string{zStr}, + SPtrs: []*string{&zStr}, + Children: []Child{{}}, + ChildPtr: new(Child), + ChildToEmbed: ChildToEmbed{}, + } + if !reflect.DeepEqual(v, want) { + t.Fatalf("zero: Unmarshal:\nhave: %#+v\nwant: %#+v", v, want) + } + + // Test with a pre-populated dst. + // Multiple addressable copies, as pointer-to fields will replace value during unmarshal. + vBytes0, vInt0, vStr0, vFloat0, vBool0 := []byte("x"), 1, "x", float32(1), true + vBytes1, vInt1, vStr1, vFloat1, vBool1 := []byte("x"), 1, "x", float32(1), true + vInt2, vStr2, vFloat2, vBool2 := 1, "x", float32(1), true + v = &Parent{ + I: vInt0, + IPtr: &vInt1, + Is: []int{vInt0}, + IPtrs: []*int{&vInt2}, + F: vFloat0, + FPtr: &vFloat1, + Fs: []float32{vFloat0}, + FPtrs: []*float32{&vFloat2}, + B: vBool0, + BPtr: &vBool1, + Bs: []bool{vBool0}, + BPtrs: []*bool{&vBool2}, + Bytes: vBytes0, + BytesPtr: &vBytes1, + S: vStr0, + SPtr: &vStr1, + Ss: []string{vStr0}, + SPtrs: []*string{&vStr2}, + MyI: MyInt(vInt0), + Child: Child{G: struct{ I int }{I: vInt0}}, + Children: []Child{{G: struct{ I int }{I: vInt0}}}, + ChildPtr: &Child{G: struct{ I int }{I: vInt0}}, + ChildToEmbed: ChildToEmbed{X: vBool0}, + } + if err := Unmarshal([]byte(emptyXML), v); err != nil { + t.Fatalf("populated: Unmarshal failed: got %v", err) + } + + want = &Parent{ + IPtr: &zInt, + Is: []int{vInt0, zInt}, + IPtrs: []*int{&vInt0, &zInt}, + FPtr: &zFloat, + Fs: []float32{vFloat0, zFloat}, + FPtrs: []*float32{&vFloat0, &zFloat}, + BPtr: &zBool, + Bs: []bool{vBool0, zBool}, + BPtrs: []*bool{&vBool0, &zBool}, + Bytes: []byte{}, + BytesPtr: &zBytes, + SPtr: &zStr, + Ss: []string{vStr0, zStr}, + SPtrs: []*string{&vStr0, &zStr}, + Child: Child{G: struct{ I int }{I: vInt0}}, // I should == zInt0? (zero value) + Children: []Child{{G: struct{ I int }{I: vInt0}}, {}}, + ChildPtr: &Child{G: struct{ I int }{I: vInt0}}, // I should == zInt0? (zero value) + } + if !reflect.DeepEqual(v, want) { + t.Fatalf("populated: Unmarshal:\nhave: %#+v\nwant: %#+v", v, want) + } +} diff --git a/libgo/go/encoding/xml/typeinfo.go b/libgo/go/encoding/xml/typeinfo.go index 6623c78..751caa9 100644 --- a/libgo/go/encoding/xml/typeinfo.go +++ b/libgo/go/encoding/xml/typeinfo.go @@ -42,21 +42,18 @@ const ( fMode = fElement | fAttr | fCDATA | fCharData | fInnerXml | fComment | fAny ) -var tinfoMap = make(map[reflect.Type]*typeInfo) -var tinfoLock sync.RWMutex +var tinfoMap sync.Map // map[reflect.Type]*typeInfo var nameType = reflect.TypeOf(Name{}) // getTypeInfo returns the typeInfo structure with details necessary // for marshaling and unmarshaling typ. func getTypeInfo(typ reflect.Type) (*typeInfo, error) { - tinfoLock.RLock() - tinfo, ok := tinfoMap[typ] - tinfoLock.RUnlock() - if ok { - return tinfo, nil + if ti, ok := tinfoMap.Load(typ); ok { + return ti.(*typeInfo), nil } - tinfo = &typeInfo{} + + tinfo := &typeInfo{} if typ.Kind() == reflect.Struct && typ != nameType { n := typ.NumField() for i := 0; i < n; i++ { @@ -105,10 +102,9 @@ func getTypeInfo(typ reflect.Type) (*typeInfo, error) { } } } - tinfoLock.Lock() - tinfoMap[typ] = tinfo - tinfoLock.Unlock() - return tinfo, nil + + ti, _ := tinfoMap.LoadOrStore(typ, tinfo) + return ti.(*typeInfo), nil } // structFieldInfo builds and returns a fieldInfo for f. diff --git a/libgo/go/encoding/xml/xml_test.go b/libgo/go/encoding/xml/xml_test.go index f43a5e7..dad6ed9 100644 --- a/libgo/go/encoding/xml/xml_test.go +++ b/libgo/go/encoding/xml/xml_test.go @@ -797,37 +797,3 @@ func TestIssue12417(t *testing.T) { } } } - -func TestIssue19333(t *testing.T) { - type X struct { - XMLName Name `xml:"X"` - A int `xml:",attr"` - C int - } - - var tests = []struct { - input string - ok bool - }{ - {`<X></X>`, true}, - {`<X A=""></X>`, true}, - {`<X A="bad"></X>`, true}, - {`<X></X>`, true}, - {`<X><C></C></X>`, false}, - {`<X><C/></X>`, false}, - {`<X><C>bad</C></X>`, false}, - } - - for _, tt := range tests { - err := Unmarshal([]byte(tt.input), new(X)) - if tt.ok { - if err != nil { - t.Errorf("%s: unexpected error: %v", tt.input, err) - } - } else { - if err == nil { - t.Errorf("%s: unexpected success", tt.input) - } - } - } -} |