diff options
Diffstat (limited to 'libgo/go/encoding')
32 files changed, 879 insertions, 555 deletions
diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go index 26868a3..1ed357a 100644 --- a/libgo/go/encoding/asn1/asn1.go +++ b/libgo/go/encoding/asn1/asn1.go @@ -250,7 +250,7 @@ func (oi ObjectIdentifier) String() string { // parseObjectIdentifier parses an OBJECT IDENTIFIER from the given bytes and // returns it. An object identifier is a sequence of variable length integers // that are assigned in a hierarchy. -func parseObjectIdentifier(bytes []byte) (s []int, err error) { +func parseObjectIdentifier(bytes []byte) (s ObjectIdentifier, err error) { if len(bytes) == 0 { err = SyntaxError{"zero length OBJECT IDENTIFIER"} return @@ -793,6 +793,12 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam matchAnyClassAndTag = false } + if !params.explicit && params.private && params.tag != nil { + expectedClass = ClassPrivate + expectedTag = *params.tag + matchAnyClassAndTag = false + } + // We have unwrapped any explicit tagging at this point. if !matchAnyClassAndTag && (t.class != expectedClass || t.tag != expectedTag) || (!matchAny && t.isCompound != compoundType) { @@ -1028,6 +1034,7 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { // The following tags on struct fields have special meaning to Unmarshal: // // application specifies that an APPLICATION tag is used +// private specifies that a PRIVATE tag is used // default:x sets the default value for optional integer fields (only used if optional is also present) // explicit specifies that an additional, explicit tag wraps the implicit one // optional marks the field as ASN.1 OPTIONAL diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go index 5e67dc5..f0a54e0 100644 --- a/libgo/go/encoding/asn1/asn1_test.go +++ b/libgo/go/encoding/asn1/asn1_test.go @@ -227,7 +227,7 @@ func TestBitStringRightAlign(t *testing.T) { type objectIdentifierTest struct { in []byte ok bool - out []int + out ObjectIdentifier // has base type[]int } var objectIdentifierTestData = []objectIdentifierTest{ @@ -428,11 +428,12 @@ var parseFieldParametersTestData []parseFieldParametersTest = []parseFieldParame {"optional", fieldParameters{optional: true}}, {"explicit", fieldParameters{explicit: true, tag: new(int)}}, {"application", fieldParameters{application: true, tag: new(int)}}, + {"private", fieldParameters{private: true, tag: new(int)}}, {"optional,explicit", fieldParameters{optional: true, explicit: true, tag: new(int)}}, {"default:42", fieldParameters{defaultValue: newInt64(42)}}, {"tag:17", fieldParameters{tag: newInt(17)}}, {"optional,explicit,default:42,tag:17", fieldParameters{optional: true, explicit: true, defaultValue: newInt64(42), tag: newInt(17)}}, - {"optional,explicit,default:42,tag:17,rubbish1", fieldParameters{true, true, false, newInt64(42), newInt(17), 0, 0, false, false}}, + {"optional,explicit,default:42,tag:17,rubbish1", fieldParameters{optional: true, explicit: true, application: false, defaultValue: newInt64(42), tag: newInt(17), stringType: 0, timeType: 0, set: false, omitEmpty: false}}, {"set", fieldParameters{set: true}}, } @@ -1079,6 +1080,7 @@ func TestTaggedRawValue(t *testing.T) { {true, []byte{0x30, 3, (ClassContextSpecific << 6) | tag, 1, 1}}, {true, []byte{0x30, 3, (ClassContextSpecific << 6) | tag | isCompound, 1, 1}}, {false, []byte{0x30, 3, (ClassApplication << 6) | tag | isCompound, 1, 1}}, + {false, []byte{0x30, 3, (ClassPrivate << 6) | tag | isCompound, 1, 1}}, } for i, test := range tests { diff --git a/libgo/go/encoding/asn1/common.go b/libgo/go/encoding/asn1/common.go index a6589a5..255d1eb 100644 --- a/libgo/go/encoding/asn1/common.go +++ b/libgo/go/encoding/asn1/common.go @@ -75,6 +75,7 @@ type fieldParameters struct { optional bool // true iff the field is OPTIONAL explicit bool // true iff an EXPLICIT tag is in use. application bool // true iff an APPLICATION tag is in use. + private bool // true iff a PRIVATE tag is in use. defaultValue *int64 // a default value for INTEGER typed fields (maybe nil). tag *int // the EXPLICIT or IMPLICIT tag (maybe nil). stringType int // the string tag to use when marshaling. @@ -130,6 +131,11 @@ func parseFieldParameters(str string) (ret fieldParameters) { if ret.tag == nil { ret.tag = new(int) } + case part == "private": + ret.private = true + if ret.tag == nil { + ret.tag = new(int) + } case part == "omitempty": ret.omitEmpty = true } diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index 3e85651..c9ae2ca 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -631,6 +631,8 @@ func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { if params.tag != nil { if params.application { class = ClassApplication + } else if params.private { + class = ClassPrivate } else { class = ClassContextSpecific } diff --git a/libgo/go/encoding/asn1/marshal_test.go b/libgo/go/encoding/asn1/marshal_test.go index 4f755a1..a77826a 100644 --- a/libgo/go/encoding/asn1/marshal_test.go +++ b/libgo/go/encoding/asn1/marshal_test.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/hex" "math/big" + "reflect" "strings" "testing" "time" @@ -80,6 +81,13 @@ type applicationTest struct { B int `asn1:"application,tag:1,explicit"` } +type privateTest struct { + A int `asn1:"private,tag:0"` + B int `asn1:"private,tag:1,explicit"` + C int `asn1:"private,tag:31"` // tag size should be 2 octet + D int `asn1:"private,tag:128"` // tag size should be 3 octet +} + type numericStringTest struct { A string `asn1:"numeric"` } @@ -169,6 +177,7 @@ var marshalTests = []marshalTest{ {defaultTest{1}, "3000"}, {defaultTest{2}, "3003020102"}, {applicationTest{1, 2}, "30084001016103020102"}, + {privateTest{1, 2, 3, 4}, "3011c00101e103020102df1f0103df81000104"}, {numericStringTest{"1 9"}, "30051203312039"}, } @@ -195,6 +204,7 @@ type marshalWithParamsTest struct { var marshalWithParamsTests = []marshalWithParamsTest{ {intStruct{10}, "set", "310302010a"}, {intStruct{10}, "application", "600302010a"}, + {intStruct{10}, "private", "e00302010a"}, } func TestMarshalWithParams(t *testing.T) { @@ -244,6 +254,62 @@ func TestInvalidUTF8(t *testing.T) { } } +func TestMarshalOID(t *testing.T) { + var marshalTestsOID = []marshalTest{ + {[]byte("\x06\x01\x30"), "0403060130"}, // bytes format returns a byte sequence \x04 + // {ObjectIdentifier([]int{0}), "060100"}, // returns an error as OID 0.0 has the same encoding + {[]byte("\x06\x010"), "0403060130"}, // same as above "\x06\x010" = "\x06\x01" + "0" + {ObjectIdentifier([]int{2, 999, 3}), "0603883703"}, // Example of ITU-T X.690 + {ObjectIdentifier([]int{0, 0}), "060100"}, // zero OID + } + for i, test := range marshalTestsOID { + data, err := Marshal(test.in) + if err != nil { + t.Errorf("#%d failed: %s", i, err) + } + out, _ := hex.DecodeString(test.out) + if !bytes.Equal(out, data) { + t.Errorf("#%d got: %x want %x\n\t%q\n\t%q", i, data, out, data, out) + } + } +} + +func TestIssue11130(t *testing.T) { + data := []byte("\x06\x010") // == \x06\x01\x30 == OID = 0 (the figure) + var v interface{} + // v has Zero value here and Elem() would panic + _, err := Unmarshal(data, &v) + if err != nil { + t.Errorf("%v", err) + return + } + if reflect.TypeOf(v).String() != reflect.TypeOf(ObjectIdentifier{}).String() { + t.Errorf("marshal OID returned an invalid type") + return + } + + data1, err := Marshal(v) + if err != nil { + t.Errorf("%v", err) + return + } + + if !bytes.Equal(data, data1) { + t.Errorf("got: %q, want: %q \n", data1, data) + return + } + + var v1 interface{} + _, err = Unmarshal(data1, &v1) + if err != nil { + t.Errorf("%v", err) + return + } + if !reflect.DeepEqual(v, v1) { + t.Errorf("got: %#v data=%q , want : %#v data=%q\n ", v1, data1, v, data) + } +} + func BenchmarkMarshal(b *testing.B) { b.ReportAllocs() diff --git a/libgo/go/encoding/base32/base32.go b/libgo/go/encoding/base32/base32.go index e72ba74..3fb6cac 100644 --- a/libgo/go/encoding/base32/base32.go +++ b/libgo/go/encoding/base32/base32.go @@ -21,7 +21,7 @@ import ( // introduced for SASL GSSAPI and standardized in RFC 4648. // The alternate "base32hex" encoding is used in DNSSEC. type Encoding struct { - encode string + encode [32]byte decodeMap [256]byte padChar rune } @@ -37,8 +37,12 @@ const encodeHex = "0123456789ABCDEFGHIJKLMNOPQRSTUV" // NewEncoding returns a new Encoding defined by the given alphabet, // which must be a 32-byte string. func NewEncoding(encoder string) *Encoding { + if len(encoder) != 32 { + panic("encoding alphabet is not 32-bytes long") + } + e := new(Encoding) - e.encode = encoder + copy(e.encode[:], encoder) e.padChar = StdPadding for i := 0; i < len(e.decodeMap); i++ { @@ -96,10 +100,6 @@ func (enc Encoding) WithPadding(padding rune) *Encoding { // so Encode is not appropriate for use on individual blocks // of a large data stream. Use NewEncoder() instead. func (enc *Encoding) Encode(dst, src []byte) { - if len(src) == 0 { - return - } - for len(src) > 0 { var b [8]byte @@ -133,17 +133,17 @@ func (enc *Encoding) Encode(dst, src []byte) { size := len(dst) if size >= 8 { // Common case, unrolled for extra performance - dst[0] = enc.encode[b[0]] - dst[1] = enc.encode[b[1]] - dst[2] = enc.encode[b[2]] - dst[3] = enc.encode[b[3]] - dst[4] = enc.encode[b[4]] - dst[5] = enc.encode[b[5]] - dst[6] = enc.encode[b[6]] - dst[7] = enc.encode[b[7]] + dst[0] = enc.encode[b[0]&31] + dst[1] = enc.encode[b[1]&31] + dst[2] = enc.encode[b[2]&31] + dst[3] = enc.encode[b[3]&31] + dst[4] = enc.encode[b[4]&31] + dst[5] = enc.encode[b[5]&31] + dst[6] = enc.encode[b[6]&31] + dst[7] = enc.encode[b[7]&31] } else { for i := 0; i < size; i++ { - dst[i] = enc.encode[b[i]] + dst[i] = enc.encode[b[i]&31] } } @@ -244,8 +244,9 @@ func (e *encoder) Close() error { // If there's anything left in the buffer, flush it out if e.err == nil && e.nbuf > 0 { e.enc.Encode(e.out[0:], e.buf[0:e.nbuf]) + encodedLen := e.enc.EncodedLen(e.nbuf) e.nbuf = 0 - _, e.err = e.w.Write(e.out[0:8]) + _, e.err = e.w.Write(e.out[0:encodedLen]) } return e.err } @@ -403,15 +404,22 @@ type decoder struct { outbuf [1024 / 8 * 5]byte } -func readEncodedData(r io.Reader, buf []byte, min int) (n int, err error) { +func readEncodedData(r io.Reader, buf []byte, min int, expectsPadding bool) (n int, err error) { for n < min && err == nil { var nn int nn, err = r.Read(buf[n:]) n += nn } + // data was read, less than min bytes could be read if n < min && n > 0 && err == io.EOF { err = io.ErrUnexpectedEOF } + // no data was read, the buffer already contains some data + // when padding is disabled this is not an error, as the message can be of + // any length + if expectsPadding && min < 8 && n == 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + } return } @@ -439,15 +447,32 @@ func (d *decoder) Read(p []byte) (n int, err error) { nn = len(d.buf) } - nn, d.err = readEncodedData(d.r, d.buf[d.nbuf:nn], 8-d.nbuf) + // Minimum amount of bytes that needs to be read each cycle + var min int + var expectsPadding bool + if d.enc.padChar == NoPadding { + min = 1 + expectsPadding = false + } else { + min = 8 - d.nbuf + expectsPadding = true + } + + nn, d.err = readEncodedData(d.r, d.buf[d.nbuf:nn], min, expectsPadding) d.nbuf += nn - if d.nbuf < 8 { + if d.nbuf < min { return 0, d.err } // Decode chunk into p, or d.out and then p if p is too small. - nr := d.nbuf / 8 * 8 - nw := d.nbuf / 8 * 5 + var nr int + if d.enc.padChar == NoPadding { + nr = d.nbuf + } else { + nr = d.nbuf / 8 * 8 + } + nw := d.enc.DecodedLen(d.nbuf) + if nw > len(p) { nw, d.end, err = d.enc.decode(d.outbuf[0:], d.buf[0:nr]) d.out = d.outbuf[0:nw] diff --git a/libgo/go/encoding/base32/base32_test.go b/libgo/go/encoding/base32/base32_test.go index 56b229d..c5506ed 100644 --- a/libgo/go/encoding/base32/base32_test.go +++ b/libgo/go/encoding/base32/base32_test.go @@ -530,6 +530,86 @@ func TestDecodeWithWrongPadding(t *testing.T) { } } +func TestBufferedDecodingSameError(t *testing.T) { + testcases := []struct { + prefix string + chunkCombinations [][]string + expected error + }{ + // NBSWY3DPO5XXE3DE == helloworld + // Test with "ZZ" as extra input + {"helloworld", [][]string{ + []string{"NBSW", "Y3DP", "O5XX", "E3DE", "ZZ"}, + []string{"NBSWY3DPO5XXE3DE", "ZZ"}, + []string{"NBSWY3DPO5XXE3DEZZ"}, + []string{"NBS", "WY3", "DPO", "5XX", "E3D", "EZZ"}, + []string{"NBSWY3DPO5XXE3", "DEZZ"}, + }, io.ErrUnexpectedEOF}, + + // Test with "ZZY" as extra input + {"helloworld", [][]string{ + []string{"NBSW", "Y3DP", "O5XX", "E3DE", "ZZY"}, + []string{"NBSWY3DPO5XXE3DE", "ZZY"}, + []string{"NBSWY3DPO5XXE3DEZZY"}, + []string{"NBS", "WY3", "DPO", "5XX", "E3D", "EZZY"}, + []string{"NBSWY3DPO5XXE3", "DEZZY"}, + }, io.ErrUnexpectedEOF}, + + // Normal case, this is valid input + {"helloworld", [][]string{ + []string{"NBSW", "Y3DP", "O5XX", "E3DE"}, + []string{"NBSWY3DPO5XXE3DE"}, + []string{"NBS", "WY3", "DPO", "5XX", "E3D", "E"}, + []string{"NBSWY3DPO5XXE3", "DE"}, + }, nil}, + + // MZXW6YTB = fooba + {"fooba", [][]string{ + []string{"MZXW6YTBZZ"}, + []string{"MZXW6YTBZ", "Z"}, + []string{"MZXW6YTB", "ZZ"}, + []string{"MZXW6YT", "BZZ"}, + []string{"MZXW6Y", "TBZZ"}, + []string{"MZXW6Y", "TB", "ZZ"}, + []string{"MZXW6", "YTBZZ"}, + []string{"MZXW6", "YTB", "ZZ"}, + []string{"MZXW6", "YT", "BZZ"}, + }, io.ErrUnexpectedEOF}, + + // Normal case, this is valid input + {"fooba", [][]string{ + []string{"MZXW6YTB"}, + []string{"MZXW6YT", "B"}, + []string{"MZXW6Y", "TB"}, + []string{"MZXW6", "YTB"}, + []string{"MZXW6", "YT", "B"}, + []string{"MZXW", "6YTB"}, + []string{"MZXW", "6Y", "TB"}, + }, nil}, + } + + for _, testcase := range testcases { + for _, chunks := range testcase.chunkCombinations { + pr, pw := io.Pipe() + + // Write the encoded chunks into the pipe + go func() { + for _, chunk := range chunks { + pw.Write([]byte(chunk)) + } + pw.Close() + }() + + decoder := NewDecoder(StdEncoding, pr) + _, err := ioutil.ReadAll(decoder) + + if err != testcase.expected { + t.Errorf("Expected %v, got %v; case %s %+v", testcase.expected, err, testcase.prefix, chunks) + } + } + } +} + func TestEncodedDecodedLen(t *testing.T) { type test struct { in int @@ -578,3 +658,94 @@ func TestEncodedDecodedLen(t *testing.T) { }) } } + +func TestWithoutPaddingClose(t *testing.T) { + encodings := []*Encoding{ + StdEncoding, + StdEncoding.WithPadding(NoPadding), + } + + for _, encoding := range encodings { + for _, testpair := range pairs { + + var buf bytes.Buffer + encoder := NewEncoder(encoding, &buf) + encoder.Write([]byte(testpair.decoded)) + encoder.Close() + + expected := testpair.encoded + if encoding.padChar == NoPadding { + expected = strings.Replace(expected, "=", "", -1) + } + + res := buf.String() + + if res != expected { + t.Errorf("Expected %s got %s; padChar=%d", expected, res, encoding.padChar) + } + } + } +} + +func TestDecodeReadAll(t *testing.T) { + encodings := []*Encoding{ + StdEncoding, + StdEncoding.WithPadding(NoPadding), + } + + for _, pair := range pairs { + for encIndex, encoding := range encodings { + encoded := pair.encoded + if encoding.padChar == NoPadding { + encoded = strings.Replace(encoded, "=", "", -1) + } + + decReader, err := ioutil.ReadAll(NewDecoder(encoding, strings.NewReader(encoded))) + if err != nil { + t.Errorf("NewDecoder error: %v", err) + } + + if pair.decoded != string(decReader) { + t.Errorf("Expected %s got %s; Encoding %d", pair.decoded, decReader, encIndex) + } + } + } +} + +func TestDecodeSmallBuffer(t *testing.T) { + encodings := []*Encoding{ + StdEncoding, + StdEncoding.WithPadding(NoPadding), + } + + for bufferSize := 1; bufferSize < 200; bufferSize++ { + for _, pair := range pairs { + for encIndex, encoding := range encodings { + encoded := pair.encoded + if encoding.padChar == NoPadding { + encoded = strings.Replace(encoded, "=", "", -1) + } + + decoder := NewDecoder(encoding, strings.NewReader(encoded)) + + var allRead []byte + + for { + buf := make([]byte, bufferSize) + n, err := decoder.Read(buf) + allRead = append(allRead, buf[0:n]...) + if err == io.EOF { + break + } + if err != nil { + t.Error(err) + } + } + + if pair.decoded != string(allRead) { + t.Errorf("Expected %s got %s; Encoding %d; bufferSize %d", pair.decoded, allRead, encIndex, bufferSize) + } + } + } + } +} diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go index 9f5c493..f019654 100644 --- a/libgo/go/encoding/base64/base64_test.go +++ b/libgo/go/encoding/base64/base64_test.go @@ -159,7 +159,7 @@ func TestDecode(t *testing.T) { dbuf, err = tt.enc.DecodeString(encoded) testEqual(t, "DecodeString(%q) = error %v, want %v", encoded, err, error(nil)) - testEqual(t, "DecodeString(%q) = %q, want %q", string(dbuf), p.decoded) + testEqual(t, "DecodeString(%q) = %q, want %q", encoded, string(dbuf), p.decoded) } } } diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index 2d01a3c..85b3bc2 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -419,70 +419,71 @@ func sizeof(t reflect.Type) int { } type coder struct { - order ByteOrder - buf []byte + order ByteOrder + buf []byte + offset int } type decoder coder type encoder coder func (d *decoder) bool() bool { - x := d.buf[0] - d.buf = d.buf[1:] + x := d.buf[d.offset] + d.offset++ return x != 0 } func (e *encoder) bool(x bool) { if x { - e.buf[0] = 1 + e.buf[e.offset] = 1 } else { - e.buf[0] = 0 + e.buf[e.offset] = 0 } - e.buf = e.buf[1:] + e.offset++ } func (d *decoder) uint8() uint8 { - x := d.buf[0] - d.buf = d.buf[1:] + x := d.buf[d.offset] + d.offset++ return x } func (e *encoder) uint8(x uint8) { - e.buf[0] = x - e.buf = e.buf[1:] + e.buf[e.offset] = x + e.offset++ } func (d *decoder) uint16() uint16 { - x := d.order.Uint16(d.buf[0:2]) - d.buf = d.buf[2:] + x := d.order.Uint16(d.buf[d.offset : d.offset+2]) + d.offset += 2 return x } func (e *encoder) uint16(x uint16) { - e.order.PutUint16(e.buf[0:2], x) - e.buf = e.buf[2:] + e.order.PutUint16(e.buf[e.offset:e.offset+2], x) + e.offset += 2 } func (d *decoder) uint32() uint32 { - x := d.order.Uint32(d.buf[0:4]) - d.buf = d.buf[4:] + x := d.order.Uint32(d.buf[d.offset : d.offset+4]) + d.offset += 4 return x } func (e *encoder) uint32(x uint32) { - e.order.PutUint32(e.buf[0:4], x) - e.buf = e.buf[4:] + e.order.PutUint32(e.buf[e.offset:e.offset+4], x) + e.offset += 4 } func (d *decoder) uint64() uint64 { - x := d.order.Uint64(d.buf[0:8]) - d.buf = d.buf[8:] + x := d.order.Uint64(d.buf[d.offset : d.offset+8]) + d.offset += 8 return x } func (e *encoder) uint64(x uint64) { - e.order.PutUint64(e.buf[0:8], x) - e.buf = e.buf[8:] + e.order.PutUint64(e.buf[e.offset:e.offset+8], x) + e.offset += 8 } func (d *decoder) int8() int8 { return int8(d.uint8()) } @@ -646,15 +647,16 @@ func (e *encoder) value(v reflect.Value) { } func (d *decoder) skip(v reflect.Value) { - d.buf = d.buf[dataSize(v):] + d.offset += dataSize(v) } func (e *encoder) skip(v reflect.Value) { n := dataSize(v) - for i := range e.buf[0:n] { - e.buf[i] = 0 + zero := e.buf[e.offset : e.offset+n] + for i := range zero { + zero[i] = 0 } - e.buf = e.buf[n:] + e.offset += n } // intDataSize returns the size of the data required to represent the data when encoded. @@ -663,6 +665,8 @@ func intDataSize(data interface{}) int { switch data := data.(type) { case bool, int8, uint8, *bool, *int8, *uint8: return 1 + case []bool: + return len(data) case []int8: return len(data) case []uint8: diff --git a/libgo/go/encoding/csv/reader.go b/libgo/go/encoding/csv/reader.go index 2efc7ad..a2fd4c0 100644 --- a/libgo/go/encoding/csv/reader.go +++ b/libgo/go/encoding/csv/reader.go @@ -91,7 +91,7 @@ var ( var errInvalidDelim = errors.New("csv: invalid field or comment delimiter") func validDelim(r rune) bool { - return r != 0 && r != '\r' && r != '\n' && utf8.ValidRune(r) && r != utf8.RuneError + return r != 0 && r != '"' && r != '\r' && r != '\n' && utf8.ValidRune(r) && r != utf8.RuneError } // A Reader reads records from a CSV-encoded file. diff --git a/libgo/go/encoding/csv/reader_test.go b/libgo/go/encoding/csv/reader_test.go index 1fc69f9..5121791 100644 --- a/libgo/go/encoding/csv/reader_test.go +++ b/libgo/go/encoding/csv/reader_test.go @@ -359,6 +359,10 @@ x,,, Error: errInvalidDelim, }, { Name: "BadComma3", + Comma: '"', + Error: errInvalidDelim, + }, { + Name: "BadComma4", Comma: utf8.RuneError, Error: errInvalidDelim, }, { diff --git a/libgo/go/encoding/csv/writer.go b/libgo/go/encoding/csv/writer.go index ef3594e..31c4f9c 100644 --- a/libgo/go/encoding/csv/writer.go +++ b/libgo/go/encoding/csv/writer.go @@ -57,33 +57,46 @@ func (w *Writer) Write(record []string) error { } continue } + if err := w.w.WriteByte('"'); err != nil { return err } + for len(field) > 0 { + // Search for special characters. + i := strings.IndexAny(field, "\"\r\n") + if i < 0 { + i = len(field) + } + + // Copy verbatim everything before the special character. + if _, err := w.w.WriteString(field[:i]); err != nil { + return err + } + field = field[i:] - for _, r1 := range field { - var err error - switch r1 { - case '"': - _, err = w.w.WriteString(`""`) - case '\r': - if !w.UseCRLF { - err = w.w.WriteByte('\r') + // Encode the special character. + if len(field) > 0 { + var err error + switch field[0] { + case '"': + _, err = w.w.WriteString(`""`) + case '\r': + if !w.UseCRLF { + err = w.w.WriteByte('\r') + } + case '\n': + if w.UseCRLF { + _, err = w.w.WriteString("\r\n") + } else { + err = w.w.WriteByte('\n') + } } - case '\n': - if w.UseCRLF { - _, err = w.w.WriteString("\r\n") - } else { - err = w.w.WriteByte('\n') + field = field[1:] + if err != nil { + return err } - default: - _, err = w.w.WriteRune(r1) - } - if err != nil { - return err } } - if err := w.w.WriteByte('"'); err != nil { return err } diff --git a/libgo/go/encoding/csv/writer_test.go b/libgo/go/encoding/csv/writer_test.go index 8ddca0a..011f01c 100644 --- a/libgo/go/encoding/csv/writer_test.go +++ b/libgo/go/encoding/csv/writer_test.go @@ -13,7 +13,9 @@ import ( var writeTests = []struct { Input [][]string Output string + Error error UseCRLF bool + Comma rune }{ {Input: [][]string{{"abc"}}, Output: "abc\n"}, {Input: [][]string{{"abc"}}, Output: "abc\r\n", UseCRLF: true}, @@ -39,6 +41,11 @@ var writeTests = []struct { {Input: [][]string{{"a", "a", ""}}, Output: "a,a,\n"}, {Input: [][]string{{"a", "a", "a"}}, Output: "a,a,a\n"}, {Input: [][]string{{`\.`}}, Output: "\"\\.\"\n"}, + {Input: [][]string{{"x09\x41\xb4\x1c", "aktau"}}, Output: "x09\x41\xb4\x1c,aktau\n"}, + {Input: [][]string{{",x09\x41\xb4\x1c", "aktau"}}, Output: "\",x09\x41\xb4\x1c\",aktau\n"}, + {Input: [][]string{{"a", "a", ""}}, Output: "a|a|\n", Comma: '|'}, + {Input: [][]string{{",", ",", ""}}, Output: ",|,|\n", Comma: '|'}, + {Input: [][]string{{"foo"}}, Comma: '"', Error: errInvalidDelim}, } func TestWrite(t *testing.T) { @@ -46,9 +53,12 @@ func TestWrite(t *testing.T) { b := &bytes.Buffer{} f := NewWriter(b) f.UseCRLF = tt.UseCRLF + if tt.Comma != 0 { + f.Comma = tt.Comma + } err := f.WriteAll(tt.Input) - if err != nil { - t.Errorf("Unexpected error: %s\n", err) + if err != tt.Error { + t.Errorf("Unexpected error:\ngot %v\nwant %v", err, tt.Error) } out := b.String() if out != tt.Output { diff --git a/libgo/go/encoding/gob/codec_test.go b/libgo/go/encoding/gob/codec_test.go index 8f7b6f3..520afde 100644 --- a/libgo/go/encoding/gob/codec_test.go +++ b/libgo/go/encoding/gob/codec_test.go @@ -1478,6 +1478,10 @@ func TestFuzzOneByte(t *testing.T) { switch i { case 14, 167, 231, 265: // a slice length, corruptions are not handled yet. continue + case 248: + // Large map size, which currently causes an out of memory panic. + // See golang.org/issue/24308 and golang.org/issue/20221. + continue } indices = append(indices, i) } diff --git a/libgo/go/encoding/gob/dec_helpers.go b/libgo/go/encoding/gob/dec_helpers.go index 3aa038d..26eb9e4 100644 --- a/libgo/go/encoding/gob/dec_helpers.go +++ b/libgo/go/encoding/gob/dec_helpers.go @@ -1,4 +1,4 @@ -// Created by decgen --output dec_helpers.go; DO NOT EDIT +// Code generated by go run decgen.go -output dec_helpers.go; DO NOT EDIT. // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/libgo/go/encoding/gob/decgen.go b/libgo/go/encoding/gob/decgen.go index ef73f2d..bad4fe5 100644 --- a/libgo/go/encoding/gob/decgen.go +++ b/libgo/go/encoding/gob/decgen.go @@ -171,7 +171,7 @@ func main() { log.Fatal("usage: decgen [--output filename]") } var b bytes.Buffer - fmt.Fprintf(&b, "// Created by decgen --output %s; DO NOT EDIT\n", *output) + fmt.Fprintf(&b, "// Code generated by go run decgen.go -output %s; DO NOT EDIT.\n", *output) fmt.Fprint(&b, header) printMaps(&b, "Array") fmt.Fprint(&b, "\n") diff --git a/libgo/go/encoding/gob/decode.go b/libgo/go/encoding/gob/decode.go index 2da913f..d2f6c74 100644 --- a/libgo/go/encoding/gob/decode.go +++ b/libgo/go/encoding/gob/decode.go @@ -1070,14 +1070,14 @@ func (dec *Decoder) compileSingle(remoteId typeId, ut *userTypeInfo) (engine *de } // compileIgnoreSingle compiles the decoder engine for a non-struct top-level value that will be discarded. -func (dec *Decoder) compileIgnoreSingle(remoteId typeId) (engine *decEngine, err error) { - engine = new(decEngine) +func (dec *Decoder) compileIgnoreSingle(remoteId typeId) *decEngine { + engine := new(decEngine) engine.instr = make([]decInstr, 1) // one item op := dec.decIgnoreOpFor(remoteId, make(map[typeId]*decOp)) ovfl := overflow(dec.typeString(remoteId)) engine.instr[0] = decInstr{*op, 0, nil, ovfl} engine.numInstr = 1 - return + return engine } // compileDec compiles the decoder engine for a value. If the value is not a struct, @@ -1168,7 +1168,7 @@ func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, er if wire != nil && wire.StructT != nil { *enginePtr, err = dec.compileDec(wireId, userType(emptyStructType)) } else { - *enginePtr, err = dec.compileIgnoreSingle(wireId) + *enginePtr = dec.compileIgnoreSingle(wireId) } if err != nil { delete(dec.ignorerCache, wireId) diff --git a/libgo/go/encoding/gob/enc_helpers.go b/libgo/go/encoding/gob/enc_helpers.go index 804e539..c3b4ca8 100644 --- a/libgo/go/encoding/gob/enc_helpers.go +++ b/libgo/go/encoding/gob/enc_helpers.go @@ -1,4 +1,4 @@ -// Created by encgen --output enc_helpers.go; DO NOT EDIT +// Code generated by go run encgen.go -output enc_helpers.go; DO NOT EDIT. // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/libgo/go/encoding/gob/encgen.go b/libgo/go/encoding/gob/encgen.go index efdd928..0c051d2 100644 --- a/libgo/go/encoding/gob/encgen.go +++ b/libgo/go/encoding/gob/encgen.go @@ -150,7 +150,7 @@ func main() { log.Fatal("usage: encgen [--output filename]") } var b bytes.Buffer - fmt.Fprintf(&b, "// Created by encgen --output %s; DO NOT EDIT\n", *output) + fmt.Fprintf(&b, "// Code generated by go run encgen.go -output %s; DO NOT EDIT.\n", *output) fmt.Fprint(&b, header) printMaps(&b, "Array") fmt.Fprint(&b, "\n") diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index a1ca252..dc9bbcf 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -10,6 +10,7 @@ import ( "fmt" "io/ioutil" "reflect" + "runtime" "strings" "testing" ) @@ -1014,7 +1015,7 @@ type Bug4Secret struct { } // Test that a failed compilation doesn't leave around an executable encoder. -// Issue 3273. +// Issue 3723. func TestMutipleEncodingsOfBadType(t *testing.T) { x := Bug4Public{ Name: "name", @@ -1130,6 +1131,9 @@ func TestBadData(t *testing.T) { // TestHugeWriteFails tests that enormous messages trigger an error. func TestHugeWriteFails(t *testing.T) { + if runtime.GOARCH == "wasm" { + t.Skip("out of memory on wasm") + } if testing.Short() { // Requires allocating a monster, so don't do this from all.bash. t.Skip("skipping huge allocation in short mode") diff --git a/libgo/go/encoding/hex/hex.go b/libgo/go/encoding/hex/hex.go index e4df6cb..aee5aec 100644 --- a/libgo/go/encoding/hex/hex.go +++ b/libgo/go/encoding/hex/hex.go @@ -50,8 +50,8 @@ func DecodedLen(x int) int { return x / 2 } // Decode decodes src into DecodedLen(len(src)) bytes, // returning the actual number of bytes written to dst. // -// Decode expects that src contain only hexadecimal -// characters and that src should have an even length. +// Decode expects that src contains only hexadecimal +// characters and that src has even length. // If the input is malformed, Decode returns the number // of bytes decoded before the error. func Decode(dst, src []byte) (int, error) { @@ -101,10 +101,10 @@ func EncodeToString(src []byte) string { // DecodeString returns the bytes represented by the hexadecimal string s. // -// DecodeString expects that src contain only hexadecimal -// characters and that src should have an even length. -// If the input is malformed, DecodeString returns a string -// containing the bytes decoded before the error. +// DecodeString expects that src contains only hexadecimal +// characters and that src has even length. +// If the input is malformed, DecodeString returns +// the bytes decoded before the error. func DecodeString(s string) ([]byte, error) { src := []byte(s) // We can use the source slice itself as the destination @@ -211,6 +211,7 @@ type dumper struct { buf [14]byte used int // number of bytes in the current line n uint // number of bytes, total + closed bool } func toChar(b byte) byte { @@ -221,6 +222,10 @@ func toChar(b byte) byte { } func (h *dumper) Write(data []byte) (n int, err error) { + if h.closed { + return 0, errors.New("encoding/hex: dumper closed") + } + // Output lines look like: // 00000010 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d |./0123456789:;<=| // ^ offset ^ extra space ^ ASCII of line. @@ -277,6 +282,10 @@ func (h *dumper) Write(data []byte) (n int, err error) { func (h *dumper) Close() (err error) { // See the comments in Write() for the details of this format. + if h.closed { + return + } + h.closed = true if h.used == 0 { return } diff --git a/libgo/go/encoding/hex/hex_test.go b/libgo/go/encoding/hex/hex_test.go index b6bab21..6ba054e 100644 --- a/libgo/go/encoding/hex/hex_test.go +++ b/libgo/go/encoding/hex/hex_test.go @@ -188,6 +188,35 @@ func TestDumper(t *testing.T) { } } +func TestDumper_doubleclose(t *testing.T) { + var out bytes.Buffer + dumper := Dumper(&out) + + dumper.Write([]byte(`gopher`)) + dumper.Close() + dumper.Close() + dumper.Write([]byte(`gopher`)) + dumper.Close() + + expected := "00000000 67 6f 70 68 65 72 |gopher|\n" + if out.String() != expected { + t.Fatalf("got:\n%#v\nwant:\n%#v", out.String(), expected) + } +} + +func TestDumper_earlyclose(t *testing.T) { + var out bytes.Buffer + dumper := Dumper(&out) + + dumper.Close() + dumper.Write([]byte(`gopher`)) + + expected := "" + if out.String() != expected { + t.Fatalf("got:\n%#v\nwant:\n%#v", out.String(), expected) + } +} + func TestDump(t *testing.T) { var in [40]byte for i := range in { diff --git a/libgo/go/encoding/json/bench_test.go b/libgo/go/encoding/json/bench_test.go index 42439eb7..bd322db 100644 --- a/libgo/go/encoding/json/bench_test.go +++ b/libgo/go/encoding/json/bench_test.go @@ -13,9 +13,14 @@ package json import ( "bytes" "compress/gzip" + "fmt" + "internal/testenv" "io/ioutil" "os" + "reflect" + "runtime" "strings" + "sync" "testing" ) @@ -265,3 +270,66 @@ func BenchmarkUnmapped(b *testing.B) { } }) } + +func BenchmarkTypeFieldsCache(b *testing.B) { + var maxTypes int = 1e6 + if testenv.Builder() != "" { + maxTypes = 1e3 // restrict cache sizes on builders + } + + // Dynamically generate many new types. + types := make([]reflect.Type, maxTypes) + fs := []reflect.StructField{{ + Type: reflect.TypeOf(""), + Index: []int{0}, + }} + for i := range types { + fs[0].Name = fmt.Sprintf("TypeFieldsCache%d", i) + types[i] = reflect.StructOf(fs) + } + + // clearClear clears the cache. Other JSON operations, must not be running. + clearCache := func() { + fieldCache = sync.Map{} + } + + // MissTypes tests the performance of repeated cache misses. + // This measures the time to rebuild a cache of size nt. + for nt := 1; nt <= maxTypes; nt *= 10 { + ts := types[:nt] + b.Run(fmt.Sprintf("MissTypes%d", nt), func(b *testing.B) { + nc := runtime.GOMAXPROCS(0) + for i := 0; i < b.N; i++ { + clearCache() + var wg sync.WaitGroup + for j := 0; j < nc; j++ { + wg.Add(1) + go func(j int) { + for _, t := range ts[(j*len(ts))/nc : ((j+1)*len(ts))/nc] { + cachedTypeFields(t) + } + wg.Done() + }(j) + } + wg.Wait() + } + }) + } + + // HitTypes tests the performance of repeated cache hits. + // This measures the average time of each cache lookup. + for nt := 1; nt <= maxTypes; nt *= 10 { + // Pre-warm a cache of size nt. + clearCache() + for _, t := range types[:nt] { + cachedTypeFields(t) + } + b.Run(fmt.Sprintf("HitTypes%d", nt), func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + cachedTypeFields(types[0]) + } + }) + }) + } +} diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index 730fb92..0b29249 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -14,7 +14,6 @@ import ( "errors" "fmt" "reflect" - "runtime" "strconv" "unicode" "unicode/utf16" @@ -168,25 +167,20 @@ func (e *InvalidUnmarshalError) Error() string { return "json: Unmarshal(nil " + e.Type.String() + ")" } -func (d *decodeState) unmarshal(v interface{}) (err error) { - defer func() { - if r := recover(); r != nil { - if _, ok := r.(runtime.Error); ok { - panic(r) - } - err = r.(error) - } - }() - +func (d *decodeState) unmarshal(v interface{}) error { rv := reflect.ValueOf(v) if rv.Kind() != reflect.Ptr || rv.IsNil() { return &InvalidUnmarshalError{reflect.TypeOf(v)} } d.scan.reset() + d.scanWhile(scanSkipSpace) // We decode rv not rv.Elem because the Unmarshaler interface // test must be applied at the top level of the value. - d.value(rv) + err := d.value(rv) + if err != nil { + return err + } return d.savedError } @@ -210,7 +204,7 @@ func (n Number) Int64() (int64, error) { func isValidNumber(s string) bool { // This function implements the JSON numbers grammar. // See https://tools.ietf.org/html/rfc7159#section-6 - // and http://json.org/number.gif + // and https://json.org/number.gif if s == "" { return false @@ -269,9 +263,9 @@ func isValidNumber(s string) bool { // decodeState represents the state while decoding a JSON value. type decodeState struct { data []byte - off int // read offset in data + off int // next read offset in data + opcode int // last read result scan scanner - nextscan scanner // for calls to nextValue errorContext struct { // provides context for type errors Struct string Field string @@ -281,6 +275,11 @@ type decodeState struct { disallowUnknownFields bool } +// readIndex returns the position of the last byte read. +func (d *decodeState) readIndex() int { + return d.off - 1 +} + // errPhase is used for errors that should not happen unless // there is a bug in the JSON decoder or something is editing // the data slice while the decoder executes. @@ -295,11 +294,6 @@ func (d *decodeState) init(data []byte) *decodeState { return d } -// error aborts the decoding by panicking with err. -func (d *decodeState) error(err error) { - panic(d.addErrorContext(err)) -} - // saveError saves the first err it is called with, // for reporting at the end of the unmarshal. func (d *decodeState) saveError(err error) { @@ -321,95 +315,91 @@ func (d *decodeState) addErrorContext(err error) error { return err } -// next cuts off and returns the next full JSON value in d.data[d.off:]. -// The next value is known to be an object or array, not a literal. -func (d *decodeState) next() []byte { - c := d.data[d.off] - item, rest, err := nextValue(d.data[d.off:], &d.nextscan) - if err != nil { - d.error(err) +// skip scans to the end of what was started. +func (d *decodeState) skip() { + s, data, i := &d.scan, d.data, d.off + depth := len(s.parseState) + for { + op := s.step(s, data[i]) + i++ + if len(s.parseState) < depth { + d.off = i + d.opcode = op + return + } } - d.off = len(d.data) - len(rest) +} - // Our scanner has seen the opening brace/bracket - // and thinks we're still in the middle of the object. - // invent a closing brace/bracket to get it out. - if c == '{' { - d.scan.step(&d.scan, '}') +// scanNext processes the byte at d.data[d.off]. +func (d *decodeState) scanNext() { + s, data, i := &d.scan, d.data, d.off + if i < len(data) { + d.opcode = s.step(s, data[i]) + d.off = i + 1 } else { - d.scan.step(&d.scan, ']') + d.opcode = s.eof() + d.off = len(data) + 1 // mark processed EOF with len+1 } - - return item } // scanWhile processes bytes in d.data[d.off:] until it // receives a scan code not equal to op. -// It updates d.off and returns the new scan code. -func (d *decodeState) scanWhile(op int) int { - var newOp int - for { - if d.off >= len(d.data) { - newOp = d.scan.eof() - d.off = len(d.data) + 1 // mark processed EOF with len+1 - } else { - c := d.data[d.off] - d.off++ - newOp = d.scan.step(&d.scan, c) - } +func (d *decodeState) scanWhile(op int) { + s, data, i := &d.scan, d.data, d.off + for i < len(d.data) { + newOp := s.step(s, data[i]) + i++ if newOp != op { - break + d.opcode = newOp + d.off = i + return } } - return newOp -} - -// value decodes a JSON value from d.data[d.off:] into the value. -// it updates d.off to point past the decoded value. -func (d *decodeState) value(v reflect.Value) { - if !v.IsValid() { - _, rest, err := nextValue(d.data[d.off:], &d.nextscan) - if err != nil { - d.error(err) - } - d.off = len(d.data) - len(rest) - // d.scan thinks we're still at the beginning of the item. - // Feed in an empty string - the shortest, simplest value - - // so that it knows we got to the end of the value. - if d.scan.redo { - // rewind. - d.scan.redo = false - d.scan.step = stateBeginValue - } - d.scan.step(&d.scan, '"') - d.scan.step(&d.scan, '"') - - n := len(d.scan.parseState) - if n > 0 && d.scan.parseState[n-1] == parseObjectKey { - // d.scan thinks we just read an object key; finish the object - d.scan.step(&d.scan, ':') - d.scan.step(&d.scan, '"') - d.scan.step(&d.scan, '"') - d.scan.step(&d.scan, '}') - } - - return - } + d.off = len(d.data) + 1 // mark processed EOF with len+1 + d.opcode = d.scan.eof() +} - switch op := d.scanWhile(scanSkipSpace); op { +// value consumes a JSON value from d.data[d.off-1:], decoding into v, and +// reads the following byte ahead. If v is invalid, the value is discarded. +// The first byte of the value has been read already. +func (d *decodeState) value(v reflect.Value) error { + switch d.opcode { default: - d.error(errPhase) + return errPhase case scanBeginArray: - d.array(v) + if v.IsValid() { + if err := d.array(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() case scanBeginObject: - d.object(v) + if v.IsValid() { + if err := d.object(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() case scanBeginLiteral: - d.literal(v) + // All bytes inside literal return scanContinue op code. + start := d.readIndex() + d.scanWhile(scanContinue) + + if v.IsValid() { + if err := d.literalStore(d.data[start:d.readIndex()], v, false); err != nil { + return err + } + } } + return nil } type unquotedValue struct{} @@ -418,31 +408,37 @@ type unquotedValue struct{} // quoted string literal or literal null into an interface value. // If it finds anything other than a quoted string literal or null, // valueQuoted returns unquotedValue{}. -func (d *decodeState) valueQuoted() interface{} { - switch op := d.scanWhile(scanSkipSpace); op { +func (d *decodeState) valueQuoted() (interface{}, error) { + switch d.opcode { default: - d.error(errPhase) + return nil, errPhase case scanBeginArray: - d.array(reflect.Value{}) + d.skip() + d.scanNext() case scanBeginObject: - d.object(reflect.Value{}) + d.skip() + d.scanNext() case scanBeginLiteral: - switch v := d.literalInterface().(type) { + v, err := d.literalInterface() + if err != nil { + return nil, err + } + switch v.(type) { case nil, string: - return v + return v, nil } } - return unquotedValue{} + return unquotedValue{}, nil } // indirect walks down v allocating pointers as needed, // until it gets to a non-pointer. // if it encounters an Unmarshaler, indirect stops and returns that. // if decodingNull is true, indirect stops at the last pointer so it can be set to nil. -func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { +func indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { // Issue #24153 indicates that it is generally not a guaranteed property // that you may round-trip a reflect.Value by calling Value.Addr().Elem() // and expect the value to still be settable for values derived from @@ -507,26 +503,21 @@ func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, return nil, nil, v } -// array consumes an array from d.data[d.off-1:], decoding into the value v. -// the first byte of the array ('[') has been read already. -func (d *decodeState) array(v reflect.Value) { +// array consumes an array from d.data[d.off-1:], decoding into v. +// The first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) error { // Check for unmarshaler. - u, ut, pv := d.indirect(v, false) + u, ut, pv := indirect(v, false) if u != nil { - d.off-- - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return + start := d.readIndex() + d.skip() + return u.UnmarshalJSON(d.data[start:d.off]) } if ut != nil { d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) - d.off-- - d.next() - return + d.skip() + return nil } - v = pv // Check type of target. @@ -534,16 +525,19 @@ func (d *decodeState) array(v reflect.Value) { case reflect.Interface: if v.NumMethod() == 0 { // Decoding into nil interface? Switch to non-reflect code. - v.Set(reflect.ValueOf(d.arrayInterface())) - return + ai, err := d.arrayInterface() + if err != nil { + return err + } + v.Set(reflect.ValueOf(ai)) + return nil } // Otherwise it's invalid. fallthrough default: d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) - d.off-- - d.next() - return + d.skip() + return nil case reflect.Array: case reflect.Slice: break @@ -552,15 +546,11 @@ func (d *decodeState) array(v reflect.Value) { i := 0 for { // Look ahead for ] - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndArray { + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { break } - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - // Get element of array, growing if necessary. if v.Kind() == reflect.Slice { // Grow slice if necessary @@ -580,20 +570,26 @@ func (d *decodeState) array(v reflect.Value) { if i < v.Len() { // Decode into element. - d.value(v.Index(i)) + if err := d.value(v.Index(i)); err != nil { + return err + } } else { // Ran out of fixed array: skip. - d.value(reflect.Value{}) + if err := d.value(reflect.Value{}); err != nil { + return err + } } i++ // Next token must be , or ]. - op = d.scanWhile(scanSkipSpace) - if op == scanEndArray { + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { break } - if op != scanArrayValue { - d.error(errPhase) + if d.opcode != scanArrayValue { + return errPhase } } @@ -611,36 +607,37 @@ func (d *decodeState) array(v reflect.Value) { if i == 0 && v.Kind() == reflect.Slice { v.Set(reflect.MakeSlice(v.Type(), 0, 0)) } + return nil } var nullLiteral = []byte("null") var textUnmarshalerType = reflect.TypeOf(new(encoding.TextUnmarshaler)).Elem() -// object consumes an object from d.data[d.off-1:], decoding into the value v. -// the first byte ('{') of the object has been read already. -func (d *decodeState) object(v reflect.Value) { +// object consumes an object from d.data[d.off-1:], decoding into v. +// The first byte ('{') of the object has been read already. +func (d *decodeState) object(v reflect.Value) error { // Check for unmarshaler. - u, ut, pv := d.indirect(v, false) + u, ut, pv := indirect(v, false) if u != nil { - d.off-- - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return + start := d.readIndex() + d.skip() + return u.UnmarshalJSON(d.data[start:d.off]) } if ut != nil { d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return + d.skip() + return nil } v = pv // Decoding into nil interface? Switch to non-reflect code. if v.Kind() == reflect.Interface && v.NumMethod() == 0 { - v.Set(reflect.ValueOf(d.objectInterface())) - return + oi, err := d.objectInterface() + if err != nil { + return err + } + v.Set(reflect.ValueOf(oi)) + return nil } // Check type of target: @@ -659,9 +656,8 @@ func (d *decodeState) object(v reflect.Value) { default: if !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return + d.skip() + return nil } } if v.IsNil() { @@ -671,31 +667,30 @@ func (d *decodeState) object(v reflect.Value) { // ok default: d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return + d.skip() + return nil } var mapElem reflect.Value for { // Read opening " of string key or closing }. - op := d.scanWhile(scanSkipSpace) - if op == scanEndObject { + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { // closing } - can only happen on first iteration. break } - if op != scanBeginLiteral { - d.error(errPhase) + if d.opcode != scanBeginLiteral { + return errPhase } // Read key. - start := d.off - 1 - op = d.scanWhile(scanContinue) - item := d.data[start : d.off-1] + start := d.readIndex() + d.scanWhile(scanContinue) + item := d.data[start:d.readIndex()] key, ok := unquoteBytes(item) if !ok { - d.error(errPhase) + return errPhase } // Figure out field corresponding to key. @@ -756,24 +751,35 @@ func (d *decodeState) object(v reflect.Value) { } // Read : before value. - if op == scanSkipSpace { - op = d.scanWhile(scanSkipSpace) + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) } - if op != scanObjectKey { - d.error(errPhase) + if d.opcode != scanObjectKey { + return errPhase } + d.scanWhile(scanSkipSpace) if destring { - switch qv := d.valueQuoted().(type) { + q, err := d.valueQuoted() + if err != nil { + return err + } + switch qv := q.(type) { case nil: - d.literalStore(nullLiteral, subv, false) + if err := d.literalStore(nullLiteral, subv, false); err != nil { + return err + } case string: - d.literalStore([]byte(qv), subv, true) + if err := d.literalStore([]byte(qv), subv, true); err != nil { + return err + } default: d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) } } else { - d.value(subv) + if err := d.value(subv); err != nil { + return err + } } // Write value back to map; @@ -786,7 +792,9 @@ func (d *decodeState) object(v reflect.Value) { kv = reflect.ValueOf(key).Convert(kt) case reflect.PtrTo(kt).Implements(textUnmarshalerType): kv = reflect.New(v.Type().Key()) - d.literalStore(item, kv, true) + if err := d.literalStore(item, kv, true); err != nil { + return err + } kv = kv.Elem() default: switch kt.Kind() { @@ -795,7 +803,7 @@ func (d *decodeState) object(v reflect.Value) { n, err := strconv.ParseInt(s, 10, 64) if err != nil || reflect.Zero(kt).OverflowInt(n) { d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) - return + return nil } kv = reflect.ValueOf(n).Convert(kt) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: @@ -803,7 +811,7 @@ func (d *decodeState) object(v reflect.Value) { n, err := strconv.ParseUint(s, 10, 64) if err != nil || reflect.Zero(kt).OverflowUint(n) { d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) - return + return nil } kv = reflect.ValueOf(n).Convert(kt) default: @@ -814,32 +822,20 @@ func (d *decodeState) object(v reflect.Value) { } // Next token must be , or }. - op = d.scanWhile(scanSkipSpace) - if op == scanEndObject { + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndObject { break } - if op != scanObjectValue { - d.error(errPhase) + if d.opcode != scanObjectValue { + return errPhase } d.errorContext.Struct = "" d.errorContext.Field = "" } -} - -// literal consumes a literal from d.data[d.off-1:], decoding into the value v. -// The first byte of the literal has been read already -// (that's how the caller knows it's a literal). -func (d *decodeState) literal(v reflect.Value) { - // All bytes inside literal return scanContinue op code. - start := d.off - 1 - op := d.scanWhile(scanContinue) - - // Scan read one byte too far; back up. - d.off-- - d.scan.undo(op) - - d.literalStore(d.data[start:d.off], v, false) + return nil } // convertNumber converts the number literal s to a float64 or a Number @@ -862,21 +858,17 @@ var numberType = reflect.TypeOf(Number("")) // fromQuoted indicates whether this literal came from unwrapping a // string from the ",string" struct tag option. this is used only to // produce more helpful error messages. -func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) { +func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) error { // Check for unmarshaler. if len(item) == 0 { //Empty string given d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - return + return nil } isNull := item[0] == 'n' // null - u, ut, pv := d.indirect(v, isNull) + u, ut, pv := indirect(v, isNull) if u != nil { - err := u.UnmarshalJSON(item) - if err != nil { - d.error(err) - } - return + return u.UnmarshalJSON(item) } if ut != nil { if item[0] != '"' { @@ -892,23 +884,18 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool default: val = "number" } - d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.readIndex())}) } - return + return nil } s, ok := unquoteBytes(item) if !ok { if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(errPhase) + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) } + return errPhase } - err := ut.UnmarshalText(s) - if err != nil { - d.error(err) - } - return + return ut.UnmarshalText(s) } v = pv @@ -939,7 +926,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) } case reflect.Bool: v.SetBool(value) @@ -947,7 +934,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if v.NumMethod() == 0 { v.Set(reflect.ValueOf(value)) } else { - d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) } } @@ -955,17 +942,16 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool s, ok := unquoteBytes(item) if !ok { if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(errPhase) + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) } + return errPhase } switch v.Kind() { default: - d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) case reflect.Slice: if v.Type().Elem().Kind() != reflect.Uint8 { - d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) break } b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) @@ -981,17 +967,16 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if v.NumMethod() == 0 { v.Set(reflect.ValueOf(string(s))) } else { - d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) } } default: // number if c != '-' && (c < '0' || c > '9') { if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(errPhase) + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) } + return errPhase } s := string(item) switch v.Kind() { @@ -999,15 +984,14 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if v.Kind() == reflect.String && v.Type() == numberType { v.SetString(s) if !isValidNumber(s) { - d.error(fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item)) + return fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item) } break } if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.off)}) + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) } + return &UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())} case reflect.Interface: n, err := d.convertNumber(s) if err != nil { @@ -1015,7 +999,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool break } if v.NumMethod() != 0 { - d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) break } v.Set(reflect.ValueOf(n)) @@ -1023,7 +1007,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: n, err := strconv.ParseInt(s, 10, 64) if err != nil || v.OverflowInt(n) { - d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) break } v.SetInt(n) @@ -1031,7 +1015,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: n, err := strconv.ParseUint(s, 10, 64) if err != nil || v.OverflowUint(n) { - d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) break } v.SetUint(n) @@ -1039,12 +1023,13 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Float32, reflect.Float64: n, err := strconv.ParseFloat(s, v.Type().Bits()) if err != nil || v.OverflowFloat(n) { - d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) break } v.SetFloat(n) } } + return nil } // The xxxInterface routines build up a value to be stored @@ -1052,128 +1037,138 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool // but they avoid the weight of reflection in this common case. // valueInterface is like value but returns interface{} -func (d *decodeState) valueInterface() interface{} { - switch d.scanWhile(scanSkipSpace) { +func (d *decodeState) valueInterface() (val interface{}, err error) { + switch d.opcode { default: - d.error(errPhase) - panic("unreachable") + err = errPhase case scanBeginArray: - return d.arrayInterface() + val, err = d.arrayInterface() + d.scanNext() case scanBeginObject: - return d.objectInterface() + val, err = d.objectInterface() + d.scanNext() case scanBeginLiteral: - return d.literalInterface() + val, err = d.literalInterface() } + return } // arrayInterface is like array but returns []interface{}. -func (d *decodeState) arrayInterface() []interface{} { +func (d *decodeState) arrayInterface() ([]interface{}, error) { var v = make([]interface{}, 0) for { // Look ahead for ] - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndArray { + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { break } - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - - v = append(v, d.valueInterface()) + vi, err := d.valueInterface() + if err != nil { + return nil, err + } + v = append(v, vi) // Next token must be , or ]. - op = d.scanWhile(scanSkipSpace) - if op == scanEndArray { + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { break } - if op != scanArrayValue { - d.error(errPhase) + if d.opcode != scanArrayValue { + return nil, errPhase } } - return v + return v, nil } // objectInterface is like object but returns map[string]interface{}. -func (d *decodeState) objectInterface() map[string]interface{} { +func (d *decodeState) objectInterface() (map[string]interface{}, error) { m := make(map[string]interface{}) for { // Read opening " of string key or closing }. - op := d.scanWhile(scanSkipSpace) - if op == scanEndObject { + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { // closing } - can only happen on first iteration. break } - if op != scanBeginLiteral { - d.error(errPhase) + if d.opcode != scanBeginLiteral { + return nil, errPhase } // Read string key. - start := d.off - 1 - op = d.scanWhile(scanContinue) - item := d.data[start : d.off-1] + start := d.readIndex() + d.scanWhile(scanContinue) + item := d.data[start:d.readIndex()] key, ok := unquote(item) if !ok { - d.error(errPhase) + return nil, errPhase } // Read : before value. - if op == scanSkipSpace { - op = d.scanWhile(scanSkipSpace) + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) } - if op != scanObjectKey { - d.error(errPhase) + if d.opcode != scanObjectKey { + return nil, errPhase } + d.scanWhile(scanSkipSpace) // Read value. - m[key] = d.valueInterface() + vi, err := d.valueInterface() + if err != nil { + return nil, err + } + m[key] = vi // Next token must be , or }. - op = d.scanWhile(scanSkipSpace) - if op == scanEndObject { + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndObject { break } - if op != scanObjectValue { - d.error(errPhase) + if d.opcode != scanObjectValue { + return nil, errPhase } } - return m + return m, nil } -// literalInterface is like literal but returns an interface value. -func (d *decodeState) literalInterface() interface{} { +// literalInterface consumes and returns a literal from d.data[d.off-1:] and +// it reads the following byte ahead. The first byte of the literal has been +// read already (that's how the caller knows it's a literal). +func (d *decodeState) literalInterface() (interface{}, error) { // All bytes inside literal return scanContinue op code. - start := d.off - 1 - op := d.scanWhile(scanContinue) + start := d.readIndex() + d.scanWhile(scanContinue) - // Scan read one byte too far; back up. - d.off-- - d.scan.undo(op) - item := d.data[start:d.off] + item := d.data[start:d.readIndex()] switch c := item[0]; c { case 'n': // null - return nil + return nil, nil case 't', 'f': // true, false - return c == 't' + return c == 't', nil case '"': // string s, ok := unquote(item) if !ok { - d.error(errPhase) + return nil, errPhase } - return s + return s, nil default: // number if c != '-' && (c < '0' || c > '9') { - d.error(errPhase) + return nil, errPhase } n, err := d.convertNumber(string(item)) if err != nil { d.saveError(err) } - return n + return n, nil } } diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index fa1531f..ab83b81 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -2208,3 +2208,17 @@ func TestUnmarshalEmbeddedUnexported(t *testing.T) { } } } + +type unmarshalPanic struct{} + +func (unmarshalPanic) UnmarshalJSON([]byte) error { panic(0xdead) } + +func TestUnmarshalPanic(t *testing.T) { + defer func() { + if got := recover(); !reflect.DeepEqual(got, 0xdead) { + t.Errorf("panic() = (%T)(%v), want 0xdead", got, got) + } + }() + Unmarshal([]byte("{}"), &unmarshalPanic{}) + t.Fatalf("Unmarshal should have panicked") +} diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index 1e45e44..28ca5fe 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -17,12 +17,10 @@ import ( "fmt" "math" "reflect" - "runtime" "sort" "strconv" "strings" "sync" - "sync/atomic" "unicode" "unicode/utf8" ) @@ -157,12 +155,18 @@ import ( // an infinite recursion. // func Marshal(v interface{}) ([]byte, error) { - e := &encodeState{} + e := newEncodeState() + err := e.marshal(v, encOpts{escapeHTML: true}) if err != nil { return nil, err } - return e.Bytes(), nil + buf := append([]byte(nil), e.Bytes()...) + + e.Reset() + encodeStatePool.Put(e) + + return buf, nil } // MarshalIndent is like Marshal but applies Indent to format the output. @@ -283,24 +287,28 @@ func newEncodeState() *encodeState { return new(encodeState) } +// jsonError is an error wrapper type for internal use only. +// Panics with errors are wrapped in jsonError so that the top-level recover +// can distinguish intentional panics from this package. +type jsonError struct{ error } + func (e *encodeState) marshal(v interface{}, opts encOpts) (err error) { defer func() { if r := recover(); r != nil { - if _, ok := r.(runtime.Error); ok { + if je, ok := r.(jsonError); ok { + err = je.error + } else { panic(r) } - if s, ok := r.(string); ok { - panic(s) - } - err = r.(error) } }() e.reflectValue(reflect.ValueOf(v), opts) return nil } +// error aborts the encoding by panicking with err wrapped in jsonError. func (e *encodeState) error(err error) { - panic(err) + panic(jsonError{err}) } func isEmptyValue(v reflect.Value) bool { @@ -1229,65 +1237,22 @@ func typeFields(t reflect.Type) []field { // will be false: This condition is an error in Go and we skip all // the fields. func dominantField(fields []field) (field, bool) { - // The fields are sorted in increasing index-length order. The winner - // must therefore be one with the shortest index length. Drop all - // longer entries, which is easy: just truncate the slice. - length := len(fields[0].index) - tagged := -1 // Index of first tagged field. - for i, f := range fields { - if len(f.index) > length { - fields = fields[:i] - break - } - if f.tag { - if tagged >= 0 { - // Multiple tagged fields at the same level: conflict. - // Return no field. - return field{}, false - } - tagged = i - } - } - if tagged >= 0 { - return fields[tagged], true - } - // All remaining fields have the same length. If there's more than one, - // we have a conflict (two fields named "X" at the same level) and we - // return no field. - if len(fields) > 1 { + // The fields are sorted in increasing index-length order, then by presence of tag. + // That means that the first field is the dominant one. We need only check + // for error cases: two fields at top level, either both tagged or neither tagged. + if len(fields) > 1 && len(fields[0].index) == len(fields[1].index) && fields[0].tag == fields[1].tag { return field{}, false } return fields[0], true } -var fieldCache struct { - value atomic.Value // map[reflect.Type][]field - mu sync.Mutex // used only by writers -} +var fieldCache sync.Map // map[reflect.Type][]field // cachedTypeFields is like typeFields but uses a cache to avoid repeated work. func cachedTypeFields(t reflect.Type) []field { - m, _ := fieldCache.value.Load().(map[reflect.Type][]field) - f := m[t] - if f != nil { - return f - } - - // Compute fields without lock. - // Might duplicate effort but won't hold other computations back. - f = typeFields(t) - if f == nil { - f = []field{} - } - - fieldCache.mu.Lock() - m, _ = fieldCache.value.Load().(map[reflect.Type][]field) - newM := make(map[reflect.Type][]field, len(m)+1) - for k, v := range m { - newM[k] = v + if f, ok := fieldCache.Load(t); ok { + return f.([]field) } - newM[t] = f - fieldCache.value.Store(newM) - fieldCache.mu.Unlock() - return f + f, _ := fieldCache.LoadOrStore(t, typeFields(t)) + return f.([]field) } diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index 0f194e1..b90483c 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -981,3 +981,17 @@ func TestMarshalRawMessageValue(t *testing.T) { } } } + +type marshalPanic struct{} + +func (marshalPanic) MarshalJSON() ([]byte, error) { panic(0xdead) } + +func TestMarshalPanic(t *testing.T) { + defer func() { + if got := recover(); !reflect.DeepEqual(got, 0xdead) { + t.Errorf("panic() = (%T)(%v), want 0xdead", got, got) + } + }() + Marshal(&marshalPanic{}) + t.Error("Marshal should have panicked") +} diff --git a/libgo/go/encoding/json/number_test.go b/libgo/go/encoding/json/number_test.go index 4b86999..cc67018 100644 --- a/libgo/go/encoding/json/number_test.go +++ b/libgo/go/encoding/json/number_test.go @@ -10,7 +10,7 @@ import ( ) func TestNumberIsValid(t *testing.T) { - // From: http://stackoverflow.com/a/13340826 + // From: https://stackoverflow.com/a/13340826 var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) validTests := []string{ diff --git a/libgo/go/encoding/json/scanner.go b/libgo/go/encoding/json/scanner.go index ae34418..9e6d482 100644 --- a/libgo/go/encoding/json/scanner.go +++ b/libgo/go/encoding/json/scanner.go @@ -8,7 +8,7 @@ package json // Just about at the limit of what is reasonable to write by hand. // Some parts are a bit tedious, but overall it nicely factors out the // otherwise common code from the multiple scanning functions -// in this package (Compact, Indent, checkValid, nextValue, etc). +// in this package (Compact, Indent, checkValid, etc). // // This file starts with two simple examples using the scanner // before diving into the scanner itself. @@ -36,35 +36,6 @@ func checkValid(data []byte, scan *scanner) error { return nil } -// nextValue splits data after the next whole JSON value, -// returning that value and the bytes that follow it as separate slices. -// scan is passed in for use by nextValue to avoid an allocation. -func nextValue(data []byte, scan *scanner) (value, rest []byte, err error) { - scan.reset() - for i, c := range data { - v := scan.step(scan, c) - if v >= scanEndObject { - switch v { - // probe the scanner with a space to determine whether we will - // get scanEnd on the next character. Otherwise, if the next character - // is not a space, scanEndTop allocates a needless error. - case scanEndObject, scanEndArray: - if scan.step(scan, ' ') == scanEnd { - return data[:i+1], data[i+1:], nil - } - case scanError: - return nil, nil, scan.err - case scanEnd: - return data[:i], data[i:], nil - } - } - } - if scan.eof() == scanError { - return nil, nil, scan.err - } - return data, nil, nil -} - // A SyntaxError is a description of a JSON syntax error. type SyntaxError struct { msg string // description of error @@ -101,11 +72,6 @@ type scanner struct { // Error that happened, if any. err error - // 1-byte redo (see undo method) - redo bool - redoCode int - redoState func(*scanner, byte) int - // total bytes consumed, updated by decoder.Decode bytes int64 } @@ -151,7 +117,6 @@ func (s *scanner) reset() { s.step = stateBeginValue s.parseState = s.parseState[0:0] s.err = nil - s.redo = false s.endTop = false } @@ -184,7 +149,6 @@ func (s *scanner) pushParseState(p int) { func (s *scanner) popParseState() { n := len(s.parseState) - 1 s.parseState = s.parseState[0:n] - s.redo = false if n == 0 { s.step = stateEndTop s.endTop = true @@ -607,22 +571,3 @@ func quoteChar(c byte) string { s := strconv.Quote(string(c)) return "'" + s[1:len(s)-1] + "'" } - -// undo causes the scanner to return scanCode from the next state transition. -// This gives callers a simple 1-byte undo mechanism. -func (s *scanner) undo(scanCode int) { - if s.redo { - panic("json: invalid use of scanner") - } - s.redoCode = scanCode - s.redoState = s.step - s.step = stateRedo - s.redo = true -} - -// stateRedo helps implement the scanner's 1-byte undo. -func stateRedo(s *scanner, c byte) int { - s.redo = false - s.step = s.redoState - return s.redoCode -} diff --git a/libgo/go/encoding/json/scanner_test.go b/libgo/go/encoding/json/scanner_test.go index 0d4518a..6cdbe7d 100644 --- a/libgo/go/encoding/json/scanner_test.go +++ b/libgo/go/encoding/json/scanner_test.go @@ -200,43 +200,6 @@ func TestIndentErrors(t *testing.T) { } } -func TestNextValueBig(t *testing.T) { - initBig() - var scan scanner - item, rest, err := nextValue(jsonBig, &scan) - if err != nil { - t.Fatalf("nextValue: %s", err) - } - if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] { - t.Errorf("invalid item: %d %d", len(item), len(jsonBig)) - } - if len(rest) != 0 { - t.Errorf("invalid rest: %d", len(rest)) - } - - item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan) - if err != nil { - t.Fatalf("nextValue extra: %s", err) - } - if len(item) != len(jsonBig) { - t.Errorf("invalid item: %d %d", len(item), len(jsonBig)) - } - if string(rest) != "HELLO WORLD" { - t.Errorf("invalid rest: %d", len(rest)) - } -} - -var benchScan scanner - -func BenchmarkSkipValue(b *testing.B) { - initBig() - b.ResetTimer() - for i := 0; i < b.N; i++ { - nextValue(jsonBig, &benchScan) - } - b.SetBytes(int64(len(jsonBig))) -} - func diff(t *testing.T, a, b []byte) { for i := 0; ; i++ { if i >= len(a) || i >= len(b) || a[i] != b[i] { diff --git a/libgo/go/encoding/xml/xml.go b/libgo/go/encoding/xml/xml.go index f408623..ca05944 100644 --- a/libgo/go/encoding/xml/xml.go +++ b/libgo/go/encoding/xml/xml.go @@ -7,8 +7,8 @@ package xml // References: -// Annotated XML spec: http://www.xml.com/axml/testaxml.htm -// XML name spaces: http://www.w3.org/TR/REC-xml-names/ +// Annotated XML spec: https://www.xml.com/axml/testaxml.htm +// XML name spaces: https://www.w3.org/TR/REC-xml-names/ // TODO(rsc): // Test error handling. @@ -167,9 +167,9 @@ type Decoder struct { // // Setting: // - // d.Strict = false; - // d.AutoClose = HTMLAutoClose; - // d.Entity = HTMLEntity + // d.Strict = false + // d.AutoClose = xml.HTMLAutoClose + // d.Entity = xml.HTMLEntity // // creates a parser that can handle typical HTML. // @@ -198,7 +198,7 @@ type Decoder struct { // charset-conversion readers, converting from the provided // non-UTF-8 charset into UTF-8. If CharsetReader is nil or // returns an error, parsing stops with an error. One of the - // the CharsetReader's result values must be non-nil. + // CharsetReader's result values must be non-nil. CharsetReader func(charset string, input io.Reader) (io.Reader, error) // DefaultSpace sets the default name space used for unadorned tags, @@ -271,7 +271,7 @@ func NewTokenDecoder(t TokenReader) *Decoder { // it will return an error. // // Token implements XML name spaces as described by -// http://www.w3.org/TR/REC-xml-names/. Each of the +// https://www.w3.org/TR/REC-xml-names/. Each of the // Name structures contained in the Token has the Space // set to the URL identifying its name space when known. // If Token encounters an unrecognized name space prefix, @@ -806,18 +806,7 @@ func (d *Decoder) rawToken() (Token, error) { } d.ungetc(b) - n := len(attr) - if n >= cap(attr) { - nCap := 2 * cap(attr) - if nCap == 0 { - nCap = 4 - } - nattr := make([]Attr, n, nCap) - copy(nattr, attr) - attr = nattr - } - attr = attr[0 : n+1] - a := &attr[n] + a := Attr{} if a.Name, ok = d.nsname(); !ok { if d.err == nil { d.err = d.syntaxError("expected attribute name in element") @@ -843,6 +832,7 @@ func (d *Decoder) rawToken() (Token, error) { } a.Value = string(data) } + attr = append(attr, a) } if empty { d.needClose = true @@ -873,7 +863,7 @@ func (d *Decoder) attrval() []byte { if !ok { return nil } - // http://www.w3.org/TR/REC-html40/intro/sgmltut.html#h-3.2.2 + // https://www.w3.org/TR/REC-html40/intro/sgmltut.html#h-3.2.2 if 'a' <= b && b <= 'z' || 'A' <= b && b <= 'Z' || '0' <= b && b <= '9' || b == '_' || b == ':' || b == '-' { d.buf.WriteByte(b) @@ -1144,13 +1134,13 @@ Input: } // Decide whether the given rune is in the XML Character Range, per -// the Char production of http://www.xml.com/axml/testaxml.htm, +// the Char production of https://www.xml.com/axml/testaxml.htm, // Section 2.2 Characters. func isInCharacterRange(r rune) (inrange bool) { return r == 0x09 || r == 0x0A || r == 0x0D || - r >= 0x20 && r <= 0xDF77 || + r >= 0x20 && r <= 0xD7FF || r >= 0xE000 && r <= 0xFFFD || r >= 0x10000 && r <= 0x10FFFF } @@ -1273,7 +1263,7 @@ func isNameString(s string) bool { } // These tables were generated by cut and paste from Appendix B of -// the XML spec at http://www.xml.com/axml/testaxml.htm +// the XML spec at https://www.xml.com/axml/testaxml.htm // and then reformatting. First corresponds to (Letter | '_' | ':') // and second corresponds to NameChar. @@ -1591,7 +1581,9 @@ var second = &unicode.RangeTable{ // HTMLEntity is an entity map containing translations for the // standard HTML entity characters. -var HTMLEntity = htmlEntity +// +// See the Decoder.Strict and Decoder.Entity fields' documentation. +var HTMLEntity map[string]string = htmlEntity var htmlEntity = map[string]string{ /* @@ -1858,7 +1850,9 @@ var htmlEntity = map[string]string{ // HTMLAutoClose is the set of HTML elements that // should be considered to close automatically. -var HTMLAutoClose = htmlAutoClose +// +// See the Decoder.Strict and Decoder.Entity fields' documentation. +var HTMLAutoClose []string = htmlAutoClose var htmlAutoClose = []string{ /* @@ -1942,10 +1936,8 @@ func escapeText(w io.Writer, s []byte, escapeNewline bool) error { } last = i } - if _, err := w.Write(s[last:]); err != nil { - return err - } - return nil + _, err := w.Write(s[last:]) + return err } // EscapeString writes to p the properly escaped XML equivalent @@ -2028,10 +2020,8 @@ func emitCDATA(w io.Writer, s []byte) error { } s = s[i:] } - if _, err := w.Write(cdataEnd); err != nil { - return err - } - return nil + _, err := w.Write(cdataEnd) + return err } // procInst parses the `param="..."` or `param='...'` diff --git a/libgo/go/encoding/xml/xml_test.go b/libgo/go/encoding/xml/xml_test.go index 7a3511d..ee4ffa24 100644 --- a/libgo/go/encoding/xml/xml_test.go +++ b/libgo/go/encoding/xml/xml_test.go @@ -650,6 +650,20 @@ func TestDisallowedCharacters(t *testing.T) { } } +func TestIsInCharacterRange(t *testing.T) { + invalid := []rune{ + utf8.MaxRune + 1, + 0xD800, // surrogate min + 0xDFFF, // surrogate max + -1, + } + for _, r := range invalid { + if isInCharacterRange(r) { + t.Errorf("rune %U considered valid", r) + } + } +} + var procInstTests = []struct { input string expect [2]string |