diff options
Diffstat (limited to 'libgo/go/encoding')
22 files changed, 515 insertions, 129 deletions
diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go index fd4dd68..d809dde 100644 --- a/libgo/go/encoding/asn1/asn1.go +++ b/libgo/go/encoding/asn1/asn1.go @@ -313,6 +313,12 @@ func parseBase128Int(bytes []byte, initOffset int) (ret, offset int, err error) } ret64 <<= 7 b := bytes[offset] + // integers should be minimally encoded, so the leading octet should + // never be 0x80 + if shifted == 0 && b == 0x80 { + err = SyntaxError{"integer is not minimally encoded"} + return + } ret64 |= int64(b & 0x7f) offset++ if b&0x80 == 0 { @@ -1031,6 +1037,12 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { // Because Unmarshal uses the reflect package, the structs // being written to must use upper case field names. // +// After parsing b, any bytes that were leftover and not used to fill +// val will be returned in rest. When parsing a SEQUENCE into a struct, +// any trailing elements of the SEQUENCE that do not have matching +// fields in val will not be included in rest, as these are considered +// valid elements of the SEQUENCE and not trailing data. +// // An ASN.1 INTEGER can be written to an int, int32, int64, // or *big.Int (from the math/big package). // If the encoded value does not fit in the Go type, diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go index d5649bf..8daae97 100644 --- a/libgo/go/encoding/asn1/asn1_test.go +++ b/libgo/go/encoding/asn1/asn1_test.go @@ -1129,3 +1129,15 @@ func TestBMPString(t *testing.T) { } } } + +func TestNonMinimalEncodedOID(t *testing.T) { + h, err := hex.DecodeString("060a2a80864886f70d01010b") + if err != nil { + t.Fatalf("failed to decode from hex string: %s", err) + } + var oid ObjectIdentifier + _, err = Unmarshal(h, &oid) + if err == nil { + t.Fatalf("accepted non-minimally encoded oid") + } +} diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index c9ae2ca..0d34d5a 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -5,10 +5,12 @@ package asn1 import ( + "bytes" "errors" "fmt" "math/big" "reflect" + "sort" "time" "unicode/utf8" ) @@ -78,6 +80,48 @@ func (m multiEncoder) Encode(dst []byte) { } } +type setEncoder []encoder + +func (s setEncoder) Len() int { + var size int + for _, e := range s { + size += e.Len() + } + return size +} + +func (s setEncoder) Encode(dst []byte) { + // Per X690 Section 11.6: The encodings of the component values of a + // set-of value shall appear in ascending order, the encodings being + // compared as octet strings with the shorter components being padded + // at their trailing end with 0-octets. + // + // First we encode each element to its TLV encoding and then use + // octetSort to get the ordering expected by X690 DER rules before + // writing the sorted encodings out to dst. + l := make([][]byte, len(s)) + for i, e := range s { + l[i] = make([]byte, e.Len()) + e.Encode(l[i]) + } + + sort.Slice(l, func(i, j int) bool { + // Since we are using bytes.Compare to compare TLV encodings we + // don't need to right pad s[i] and s[j] to the same length as + // suggested in X690. If len(s[i]) < len(s[j]) the length octet of + // s[i], which is the first determining byte, will inherently be + // smaller than the length octet of s[j]. This lets us skip the + // padding step. + return bytes.Compare(l[i], l[j]) < 0 + }) + + var off int + for _, b := range l { + copy(dst[off:], b) + off += len(b) + } +} + type taggedEncoder struct { // scratch contains temporary space for encoding the tag and length of // an element in order to avoid extra allocations. @@ -511,6 +555,9 @@ func makeBody(value reflect.Value, params fieldParameters) (e encoder, err error } } + if params.set { + return setEncoder(m), nil + } return multiEncoder(m), nil } case reflect.String: @@ -618,6 +665,15 @@ func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { tag = TagSet } + // makeField can be called for a slice that should be treated as a SET + // but doesn't have params.set set, for instance when using a slice + // with the SET type name suffix. In this case getUniversalType returns + // TagSet, but makeBody doesn't know about that so will treat the slice + // as a sequence. To work around this we set params.set. + if tag == TagSet && !params.set { + params.set = true + } + t := new(taggedEncoder) t.body, err = makeBody(v, params) diff --git a/libgo/go/encoding/asn1/marshal_test.go b/libgo/go/encoding/asn1/marshal_test.go index a77826a..5290522 100644 --- a/libgo/go/encoding/asn1/marshal_test.go +++ b/libgo/go/encoding/asn1/marshal_test.go @@ -319,3 +319,60 @@ func BenchmarkMarshal(b *testing.B) { } } } + +func TestSetEncoder(t *testing.T) { + testStruct := struct { + Strings []string `asn1:"set"` + }{ + Strings: []string{"a", "aa", "b", "bb", "c", "cc"}, + } + + // Expected ordering of the SET should be: + // a, b, c, aa, bb, cc + + output, err := Marshal(testStruct) + if err != nil { + t.Errorf("%v", err) + } + + expectedOrder := []string{"a", "b", "c", "aa", "bb", "cc"} + var resultStruct struct { + Strings []string `asn1:"set"` + } + rest, err := Unmarshal(output, &resultStruct) + if err != nil { + t.Errorf("%v", err) + } + if len(rest) != 0 { + t.Error("Unmarshal returned extra garbage") + } + if !reflect.DeepEqual(expectedOrder, resultStruct.Strings) { + t.Errorf("Unexpected SET content. got: %s, want: %s", resultStruct.Strings, expectedOrder) + } +} + +func TestSetEncoderSETSliceSuffix(t *testing.T) { + type testSetSET []string + testSet := testSetSET{"a", "aa", "b", "bb", "c", "cc"} + + // Expected ordering of the SET should be: + // a, b, c, aa, bb, cc + + output, err := Marshal(testSet) + if err != nil { + t.Errorf("%v", err) + } + + expectedOrder := testSetSET{"a", "b", "c", "aa", "bb", "cc"} + var resultSet testSetSET + rest, err := Unmarshal(output, &resultSet) + if err != nil { + t.Errorf("%v", err) + } + if len(rest) != 0 { + t.Error("Unmarshal returned extra garbage") + } + if !reflect.DeepEqual(expectedOrder, resultSet) { + t.Errorf("Unexpected SET content. got: %s, want: %s", resultSet, expectedOrder) + } +} diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go index 690d3ce..0c33f8e 100644 --- a/libgo/go/encoding/base64/base64.go +++ b/libgo/go/encoding/base64/base64.go @@ -480,15 +480,16 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { si := 0 for strconv.IntSize >= 64 && len(src)-si >= 8 && len(dst)-n >= 8 { + src2 := src[si : si+8] if dn, ok := assemble64( - enc.decodeMap[src[si+0]], - enc.decodeMap[src[si+1]], - enc.decodeMap[src[si+2]], - enc.decodeMap[src[si+3]], - enc.decodeMap[src[si+4]], - enc.decodeMap[src[si+5]], - enc.decodeMap[src[si+6]], - enc.decodeMap[src[si+7]], + enc.decodeMap[src2[0]], + enc.decodeMap[src2[1]], + enc.decodeMap[src2[2]], + enc.decodeMap[src2[3]], + enc.decodeMap[src2[4]], + enc.decodeMap[src2[5]], + enc.decodeMap[src2[6]], + enc.decodeMap[src2[7]], ); ok { binary.BigEndian.PutUint64(dst[n:], dn) n += 6 @@ -504,11 +505,12 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { } for len(src)-si >= 4 && len(dst)-n >= 4 { + src2 := src[si : si+4] if dn, ok := assemble32( - enc.decodeMap[src[si+0]], - enc.decodeMap[src[si+1]], - enc.decodeMap[src[si+2]], - enc.decodeMap[src[si+3]], + enc.decodeMap[src2[0]], + enc.decodeMap[src2[1]], + enc.decodeMap[src2[2]], + enc.decodeMap[src2[3]], ); ok { binary.BigEndian.PutUint32(dst[n:], dn) n += 3 diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go index bc67036..c2c9478 100644 --- a/libgo/go/encoding/base64/base64_test.go +++ b/libgo/go/encoding/base64/base64_test.go @@ -401,7 +401,7 @@ func TestDecoderIssue3577(t *testing.T) { source: "VHdhcyBicmlsbGlnLCBhbmQgdGhlIHNsaXRoeSB0b3Zlcw==", // twas brillig... nextc: next, }) - errc := make(chan error) + errc := make(chan error, 1) go func() { _, err := ioutil.ReadAll(d) errc <- err diff --git a/libgo/go/encoding/binary/varint.go b/libgo/go/encoding/binary/varint.go index bcb8ac9..38af610 100644 --- a/libgo/go/encoding/binary/varint.go +++ b/libgo/go/encoding/binary/varint.go @@ -106,13 +106,13 @@ var overflow = errors.New("binary: varint overflows a 64-bit integer") func ReadUvarint(r io.ByteReader) (uint64, error) { var x uint64 var s uint - for i := 0; ; i++ { + for i := 0; i < MaxVarintLen64; i++ { b, err := r.ReadByte() if err != nil { return x, err } if b < 0x80 { - if i > 9 || i == 9 && b > 1 { + if i == 9 && b > 1 { return x, overflow } return x | uint64(b)<<s, nil @@ -120,6 +120,7 @@ func ReadUvarint(r io.ByteReader) (uint64, error) { x |= uint64(b&0x7f) << s s += 7 } + return x, overflow } // ReadVarint reads an encoded signed integer from r and returns it as an int64. diff --git a/libgo/go/encoding/binary/varint_test.go b/libgo/go/encoding/binary/varint_test.go index ca411ec..6ef4c99 100644 --- a/libgo/go/encoding/binary/varint_test.go +++ b/libgo/go/encoding/binary/varint_test.go @@ -121,21 +121,27 @@ func TestBufferTooSmall(t *testing.T) { } } -func testOverflow(t *testing.T, buf []byte, n0 int, err0 error) { +func testOverflow(t *testing.T, buf []byte, x0 uint64, n0 int, err0 error) { x, n := Uvarint(buf) if x != 0 || n != n0 { t.Errorf("Uvarint(%v): got x = %d, n = %d; want 0, %d", buf, x, n, n0) } - x, err := ReadUvarint(bytes.NewReader(buf)) - if x != 0 || err != err0 { - t.Errorf("ReadUvarint(%v): got x = %d, err = %s; want 0, %s", buf, x, err, err0) + r := bytes.NewReader(buf) + len := r.Len() + x, err := ReadUvarint(r) + if x != x0 || err != err0 { + t.Errorf("ReadUvarint(%v): got x = %d, err = %s; want %d, %s", buf, x, err, x0, err0) + } + if read := len - r.Len(); read > MaxVarintLen64 { + t.Errorf("ReadUvarint(%v): read more than MaxVarintLen64 bytes, got %d", buf, read) } } func TestOverflow(t *testing.T) { - testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x2}, -10, overflow) - testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1, 0, 0}, -13, overflow) + testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x2}, 0, -10, overflow) + testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1, 0, 0}, 0, -13, overflow) + testOverflow(t, []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 1<<64-1, 0, overflow) // 11 bytes, should overflow } func TestNonCanonicalZero(t *testing.T) { diff --git a/libgo/go/encoding/csv/writer.go b/libgo/go/encoding/csv/writer.go index 3f34bc5..ac64b4d 100644 --- a/libgo/go/encoding/csv/writer.go +++ b/libgo/go/encoding/csv/writer.go @@ -158,10 +158,24 @@ func (w *Writer) fieldNeedsQuotes(field string) bool { if field == "" { return false } - if field == `\.` || strings.ContainsRune(field, w.Comma) || strings.ContainsAny(field, "\"\r\n") { + + if field == `\.` { return true } + if w.Comma < utf8.RuneSelf { + for i := 0; i < len(field); i++ { + c := field[i] + if c == '\n' || c == '\r' || c == '"' || c == byte(w.Comma) { + return true + } + } + } else { + if strings.ContainsRune(field, w.Comma) || strings.ContainsAny(field, "\"\r\n") { + return true + } + } + r1, _ := utf8.DecodeRuneInString(field) return unicode.IsSpace(r1) } diff --git a/libgo/go/encoding/csv/writer_test.go b/libgo/go/encoding/csv/writer_test.go index 011f01c..ab28b0d7 100644 --- a/libgo/go/encoding/csv/writer_test.go +++ b/libgo/go/encoding/csv/writer_test.go @@ -93,3 +93,20 @@ func TestError(t *testing.T) { t.Error("Error should not be nil") } } + +var benchmarkWriteData = [][]string{ + {"abc", "def", "12356", "1234567890987654311234432141542132"}, + {"abc", "def", "12356", "1234567890987654311234432141542132"}, + {"abc", "def", "12356", "1234567890987654311234432141542132"}, +} + +func BenchmarkWrite(b *testing.B) { + for i := 0; i < b.N; i++ { + w := NewWriter(&bytes.Buffer{}) + err := w.WriteAll(benchmarkWriteData) + if err != nil { + b.Fatal(err) + } + w.Flush() + } +} diff --git a/libgo/go/encoding/hex/hex_test.go b/libgo/go/encoding/hex/hex_test.go index dbb00b9..31e3f68 100644 --- a/libgo/go/encoding/hex/hex_test.go +++ b/libgo/go/encoding/hex/hex_test.go @@ -267,7 +267,6 @@ func BenchmarkDecode(b *testing.B) { func BenchmarkDump(b *testing.B) { for _, size := range []int{256, 1024, 4096, 16384} { src := bytes.Repeat([]byte{2, 3, 5, 7, 9, 11, 13, 17}, size/8) - sink = make([]byte, 2*size) b.Run(fmt.Sprintf("%v", size), func(b *testing.B) { b.SetBytes(int64(size)) diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index b434846..86d8a69 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -213,9 +213,6 @@ type decodeState struct { savedError error useNumber bool disallowUnknownFields bool - // safeUnquote is the number of current string literal bytes that don't - // need to be unquoted. When negative, no bytes need unquoting. - safeUnquote int } // readIndex returns the position of the last byte read. @@ -317,27 +314,13 @@ func (d *decodeState) rescanLiteral() { Switch: switch data[i-1] { case '"': // string - // safeUnquote is initialized at -1, which means that all bytes - // checked so far can be unquoted at a later time with no work - // at all. When reaching the closing '"', if safeUnquote is - // still -1, all bytes can be unquoted with no work. Otherwise, - // only those bytes up until the first '\\' or non-ascii rune - // can be safely unquoted. - safeUnquote := -1 for ; i < len(data); i++ { - if c := data[i]; c == '\\' { - if safeUnquote < 0 { // first unsafe byte - safeUnquote = int(i - d.off) - } + switch data[i] { + case '\\': i++ // escaped char - } else if c == '"' { - d.safeUnquote = safeUnquote + case '"': i++ // tokenize the closing quote too break Switch - } else if c >= utf8.RuneSelf { - if safeUnquote < 0 { // first unsafe byte - safeUnquote = int(i - d.off) - } } } case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': // number @@ -691,7 +674,7 @@ func (d *decodeState) object(v reflect.Value) error { start := d.readIndex() d.rescanLiteral() item := d.data[start:d.readIndex()] - key, ok := d.unquoteBytes(item) + key, ok := unquoteBytes(item) if !ok { panic(phasePanicMsg) } @@ -892,7 +875,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.readIndex())}) return nil } - s, ok := d.unquoteBytes(item) + s, ok := unquoteBytes(item) if !ok { if fromQuoted { return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) @@ -943,7 +926,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool } case '"': // string - s, ok := d.unquoteBytes(item) + s, ok := unquoteBytes(item) if !ok { if fromQuoted { return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) @@ -1103,7 +1086,7 @@ func (d *decodeState) objectInterface() map[string]interface{} { start := d.readIndex() d.rescanLiteral() item := d.data[start:d.readIndex()] - key, ok := d.unquote(item) + key, ok := unquote(item) if !ok { panic(phasePanicMsg) } @@ -1152,7 +1135,7 @@ func (d *decodeState) literalInterface() interface{} { return c == 't' case '"': // string - s, ok := d.unquote(item) + s, ok := unquote(item) if !ok { panic(phasePanicMsg) } @@ -1195,26 +1178,38 @@ func getu4(s []byte) rune { // unquote converts a quoted JSON string literal s into an actual string t. // The rules are different than for Go, so cannot use strconv.Unquote. -// The first byte in s must be '"'. -func (d *decodeState) unquote(s []byte) (t string, ok bool) { - s, ok = d.unquoteBytes(s) +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) t = string(s) return } -func (d *decodeState) unquoteBytes(s []byte) (t []byte, ok bool) { - // We already know that s[0] == '"'. However, we don't know that the - // closing quote exists in all cases, such as when the string is nested - // via the ",string" option. - if len(s) < 2 || s[len(s)-1] != '"' { +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { return } s = s[1 : len(s)-1] - // If there are no unusual characters, no unquoting is needed, so return - // a slice of the original bytes. - r := d.safeUnquote - if r == -1 { + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { return s, true } diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index 498bd97..219e845 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -2419,7 +2419,7 @@ func (m *textUnmarshalerString) UnmarshalText(text []byte) error { return nil } -// Test unmarshal to a map, with map key is a user defined type. +// Test unmarshal to a map, where the map key is a user defined type. // See golang.org/issues/34437. func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) { var p map[textUnmarshalerString]string @@ -2428,6 +2428,147 @@ func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) { } if _, ok := p["foo"]; !ok { - t.Errorf(`Key "foo" is not existed in map: %v`, p) + t.Errorf(`Key "foo" does not exist in map: %v`, p) + } +} + +func TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) { + // See golang.org/issues/38105. + var p map[textUnmarshalerString]string + if err := Unmarshal([]byte(`{"开源":"12345开源"}`), &p); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + if _, ok := p["开源"]; !ok { + t.Errorf(`Key "开源" does not exist in map: %v`, p) + } + + // See golang.org/issues/38126. + type T struct { + F1 string `json:"F1,string"` + } + t1 := T{"aaa\tbbb"} + + b, err := Marshal(t1) + if err != nil { + t.Fatalf("Marshal unexpected error: %v", err) + } + var t2 T + if err := Unmarshal(b, &t2); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + if t1 != t2 { + t.Errorf("Marshal and Unmarshal roundtrip mismatch: want %q got %q", t1, t2) + } + + // See golang.org/issues/39555. + input := map[textUnmarshalerString]string{"FOO": "", `"`: ""} + + encoded, err := Marshal(input) + if err != nil { + t.Fatalf("Marshal unexpected error: %v", err) + } + var got map[textUnmarshalerString]string + if err := Unmarshal(encoded, &got); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + want := map[textUnmarshalerString]string{"foo": "", `"`: ""} + if !reflect.DeepEqual(want, got) { + t.Fatalf("Unexpected roundtrip result:\nwant: %q\ngot: %q", want, got) + } +} + +func TestUnmarshalMaxDepth(t *testing.T) { + testcases := []struct { + name string + data string + errMaxDepth bool + }{ + { + name: "ArrayUnderMaxNestingDepth", + data: `{"a":` + strings.Repeat(`[`, 10000-1) + strings.Repeat(`]`, 10000-1) + `}`, + errMaxDepth: false, + }, + { + name: "ArrayOverMaxNestingDepth", + data: `{"a":` + strings.Repeat(`[`, 10000) + strings.Repeat(`]`, 10000) + `}`, + errMaxDepth: true, + }, + { + name: "ArrayOverStackDepth", + data: `{"a":` + strings.Repeat(`[`, 3000000) + strings.Repeat(`]`, 3000000) + `}`, + errMaxDepth: true, + }, + { + name: "ObjectUnderMaxNestingDepth", + data: `{"a":` + strings.Repeat(`{"a":`, 10000-1) + `0` + strings.Repeat(`}`, 10000-1) + `}`, + errMaxDepth: false, + }, + { + name: "ObjectOverMaxNestingDepth", + data: `{"a":` + strings.Repeat(`{"a":`, 10000) + `0` + strings.Repeat(`}`, 10000) + `}`, + errMaxDepth: true, + }, + { + name: "ObjectOverStackDepth", + data: `{"a":` + strings.Repeat(`{"a":`, 3000000) + `0` + strings.Repeat(`}`, 3000000) + `}`, + errMaxDepth: true, + }, + } + + targets := []struct { + name string + newValue func() interface{} + }{ + { + name: "unstructured", + newValue: func() interface{} { + var v interface{} + return &v + }, + }, + { + name: "typed named field", + newValue: func() interface{} { + v := struct { + A interface{} `json:"a"` + }{} + return &v + }, + }, + { + name: "typed missing field", + newValue: func() interface{} { + v := struct { + B interface{} `json:"b"` + }{} + return &v + }, + }, + { + name: "custom unmarshaler", + newValue: func() interface{} { + v := unmarshaler{} + return &v + }, + }, + } + + for _, tc := range testcases { + for _, target := range targets { + t.Run(target.name+"-"+tc.name, func(t *testing.T) { + err := Unmarshal([]byte(tc.data), target.newValue()) + if !tc.errMaxDepth { + if err != nil { + t.Errorf("unexpected error: %v", err) + } + } else { + if err == nil { + t.Errorf("expected error containing 'exceeded max depth', got none") + } else if !strings.Contains(err.Error(), "exceeded max depth") { + t.Errorf("expected error containing 'exceeded max depth', got: %v", err) + } + } + }) + } } } diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index 39cdaeb..578d551 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -635,11 +635,12 @@ func stringEncoder(e *encodeState, v reflect.Value, opts encOpts) { return } if opts.quoted { - b := make([]byte, 0, v.Len()+2) - b = append(b, '"') - b = append(b, []byte(v.String())...) - b = append(b, '"') - e.stringBytes(b, opts.escapeHTML) + e2 := newEncodeState() + // Since we encode the string twice, we only need to escape HTML + // the first time. + e2.string(v.String(), opts.escapeHTML) + e.stringBytes(e2.Bytes(), false) + encodeStatePool.Put(e2) } else { e.string(v.String(), opts.escapeHTML) } @@ -649,7 +650,7 @@ func stringEncoder(e *encodeState, v reflect.Value, opts encOpts) { func isValidNumber(s string) bool { // This function implements the JSON numbers grammar. // See https://tools.ietf.org/html/rfc7159#section-6 - // and https://json.org/number.gif + // and https://www.json.org/img/number.png if s == "" { return false diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index 5110c7d..7290eca 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -79,37 +79,66 @@ type StringTag struct { NumberStr Number `json:",string"` } -var stringTagExpected = `{ - "BoolStr": "true", - "IntStr": "42", - "UintptrStr": "44", - "StrStr": "\"xzbit\"", - "NumberStr": "46" -}` - -func TestStringTag(t *testing.T) { - var s StringTag - s.BoolStr = true - s.IntStr = 42 - s.UintptrStr = 44 - s.StrStr = "xzbit" - s.NumberStr = "46" - got, err := MarshalIndent(&s, "", " ") - if err != nil { - t.Fatal(err) - } - if got := string(got); got != stringTagExpected { - t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected) +func TestRoundtripStringTag(t *testing.T) { + tests := []struct { + name string + in StringTag + want string // empty to just test that we roundtrip + }{ + { + name: "AllTypes", + in: StringTag{ + BoolStr: true, + IntStr: 42, + UintptrStr: 44, + StrStr: "xzbit", + NumberStr: "46", + }, + want: `{ + "BoolStr": "true", + "IntStr": "42", + "UintptrStr": "44", + "StrStr": "\"xzbit\"", + "NumberStr": "46" + }`, + }, + { + // See golang.org/issues/38173. + name: "StringDoubleEscapes", + in: StringTag{ + StrStr: "\b\f\n\r\t\"\\", + NumberStr: "0", // just to satisfy the roundtrip + }, + want: `{ + "BoolStr": "false", + "IntStr": "0", + "UintptrStr": "0", + "StrStr": "\"\\u0008\\u000c\\n\\r\\t\\\"\\\\\"", + "NumberStr": "0" + }`, + }, } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Indent with a tab prefix to make the multi-line string + // literals in the table nicer to read. + got, err := MarshalIndent(&test.in, "\t\t\t", "\t") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != test.want { + t.Fatalf(" got: %s\nwant: %s\n", got, test.want) + } - // Verify that it round-trips. - var s2 StringTag - err = NewDecoder(bytes.NewReader(got)).Decode(&s2) - if err != nil { - t.Fatalf("Decode: %v", err) - } - if !reflect.DeepEqual(s, s2) { - t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", s, string(got), s2) + // Verify that it round-trips. + var s2 StringTag + if err := Unmarshal(got, &s2); err != nil { + t.Fatalf("Decode: %v", err) + } + if !reflect.DeepEqual(test.in, s2) { + t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", test.in, string(got), s2) + } + }) } } diff --git a/libgo/go/encoding/json/scanner.go b/libgo/go/encoding/json/scanner.go index 552bd70..9dc1903 100644 --- a/libgo/go/encoding/json/scanner.go +++ b/libgo/go/encoding/json/scanner.go @@ -139,6 +139,10 @@ const ( parseArrayValue // parsing array value ) +// This limits the max nesting depth to prevent stack overflow. +// This is permitted by https://tools.ietf.org/html/rfc7159#section-9 +const maxNestingDepth = 10000 + // reset prepares the scanner for use. // It must be called before calling s.step. func (s *scanner) reset() { @@ -168,8 +172,13 @@ func (s *scanner) eof() int { } // pushParseState pushes a new parse state p onto the parse stack. -func (s *scanner) pushParseState(p int) { - s.parseState = append(s.parseState, p) +// an error state is returned if maxNestingDepth was exceeded, otherwise successState is returned. +func (s *scanner) pushParseState(c byte, newParseState int, successState int) int { + s.parseState = append(s.parseState, newParseState) + if len(s.parseState) <= maxNestingDepth { + return successState + } + return s.error(c, "exceeded max depth") } // popParseState pops a parse state (already obtained) off the stack @@ -186,12 +195,12 @@ func (s *scanner) popParseState() { } func isSpace(c byte) bool { - return c == ' ' || c == '\t' || c == '\r' || c == '\n' + return c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') } // stateBeginValueOrEmpty is the state after reading `[`. func stateBeginValueOrEmpty(s *scanner, c byte) int { - if c <= ' ' && isSpace(c) { + if isSpace(c) { return scanSkipSpace } if c == ']' { @@ -202,18 +211,16 @@ func stateBeginValueOrEmpty(s *scanner, c byte) int { // stateBeginValue is the state at the beginning of the input. func stateBeginValue(s *scanner, c byte) int { - if c <= ' ' && isSpace(c) { + if isSpace(c) { return scanSkipSpace } switch c { case '{': s.step = stateBeginStringOrEmpty - s.pushParseState(parseObjectKey) - return scanBeginObject + return s.pushParseState(c, parseObjectKey, scanBeginObject) case '[': s.step = stateBeginValueOrEmpty - s.pushParseState(parseArrayValue) - return scanBeginArray + return s.pushParseState(c, parseArrayValue, scanBeginArray) case '"': s.step = stateInString return scanBeginLiteral @@ -242,7 +249,7 @@ func stateBeginValue(s *scanner, c byte) int { // stateBeginStringOrEmpty is the state after reading `{`. func stateBeginStringOrEmpty(s *scanner, c byte) int { - if c <= ' ' && isSpace(c) { + if isSpace(c) { return scanSkipSpace } if c == '}' { @@ -255,7 +262,7 @@ func stateBeginStringOrEmpty(s *scanner, c byte) int { // stateBeginString is the state after reading `{"key": value,`. func stateBeginString(s *scanner, c byte) int { - if c <= ' ' && isSpace(c) { + if isSpace(c) { return scanSkipSpace } if c == '"' { @@ -275,7 +282,7 @@ func stateEndValue(s *scanner, c byte) int { s.endTop = true return stateEndTop(s, c) } - if c <= ' ' && isSpace(c) { + if isSpace(c) { s.step = stateEndValue return scanSkipSpace } diff --git a/libgo/go/encoding/json/stream_test.go b/libgo/go/encoding/json/stream_test.go index ebb4f23..c9e5334 100644 --- a/libgo/go/encoding/json/stream_test.go +++ b/libgo/go/encoding/json/stream_test.go @@ -144,14 +144,15 @@ func TestEncoderSetEscapeHTML(t *testing.T) { }, { "stringOption", stringOption, - `{"bar":"\"\u003chtml\u003efoobar\u003c/html\u003e\""}`, + `{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`, `{"bar":"\"<html>foobar</html>\""}`, }, } { var buf bytes.Buffer enc := NewEncoder(&buf) if err := enc.Encode(tt.v); err != nil { - t.Fatalf("Encode(%s): %s", tt.name, err) + t.Errorf("Encode(%s): %s", tt.name, err) + continue } if got := strings.TrimSpace(buf.String()); got != tt.wantEscape { t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape) @@ -159,7 +160,8 @@ func TestEncoderSetEscapeHTML(t *testing.T) { buf.Reset() enc.SetEscapeHTML(false) if err := enc.Encode(tt.v); err != nil { - t.Fatalf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err) + t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err) + continue } if got := strings.TrimSpace(buf.String()); got != tt.want { t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q", diff --git a/libgo/go/encoding/xml/marshal.go b/libgo/go/encoding/xml/marshal.go index e325f31..0554b0d 100644 --- a/libgo/go/encoding/xml/marshal.go +++ b/libgo/go/encoding/xml/marshal.go @@ -479,8 +479,11 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat xmlname := tinfo.xmlname if xmlname.name != "" { start.Name.Space, start.Name.Local = xmlname.xmlns, xmlname.name - } else if v, ok := xmlname.value(val).Interface().(Name); ok && v.Local != "" { - start.Name = v + } else { + fv := xmlname.value(val, dontInitNilPointers) + if v, ok := fv.Interface().(Name); ok && v.Local != "" { + start.Name = v + } } } if start.Name.Local == "" && finfo != nil { @@ -500,7 +503,7 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat if finfo.flags&fAttr == 0 { continue } - fv := finfo.value(val) + fv := finfo.value(val, dontInitNilPointers) if finfo.flags&fOmitEmpty != 0 && isEmptyValue(fv) { continue @@ -803,7 +806,12 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { if finfo.flags&fAttr != 0 { continue } - vf := finfo.value(val) + vf := finfo.value(val, dontInitNilPointers) + if !vf.IsValid() { + // The field is behind an anonymous struct field that's + // nil. Skip it. + continue + } switch finfo.flags & fMode { case fCDATA, fCharData: diff --git a/libgo/go/encoding/xml/marshal_test.go b/libgo/go/encoding/xml/marshal_test.go index 8473158b..31309ef 100644 --- a/libgo/go/encoding/xml/marshal_test.go +++ b/libgo/go/encoding/xml/marshal_test.go @@ -309,6 +309,11 @@ type ChardataEmptyTest struct { Contents *string `xml:",chardata"` } +type PointerAnonFields struct { + *MyInt + *NamedType +} + type MyMarshalerTest struct { } @@ -889,6 +894,18 @@ var marshalTests = []struct { `</EmbedA>`, }, + // Anonymous struct pointer field which is nil + { + Value: &EmbedB{}, + ExpectXML: `<EmbedB><FieldB></FieldB></EmbedB>`, + }, + + // Other kinds of nil anonymous fields + { + Value: &PointerAnonFields{}, + ExpectXML: `<PointerAnonFields></PointerAnonFields>`, + }, + // Test that name casing matters { Value: &NameCasing{Xy: "mixed", XY: "upper", XyA: "mixedA", XYA: "upperA"}, diff --git a/libgo/go/encoding/xml/read.go b/libgo/go/encoding/xml/read.go index 10a60ee..ef5df3f 100644 --- a/libgo/go/encoding/xml/read.go +++ b/libgo/go/encoding/xml/read.go @@ -435,7 +435,7 @@ func (d *Decoder) unmarshal(val reflect.Value, start *StartElement) error { } return UnmarshalError(e) } - fv := finfo.value(sv) + fv := finfo.value(sv, initNilPointers) if _, ok := fv.Interface().(Name); ok { fv.Set(reflect.ValueOf(start.Name)) } @@ -449,7 +449,7 @@ func (d *Decoder) unmarshal(val reflect.Value, start *StartElement) error { finfo := &tinfo.fields[i] switch finfo.flags & fMode { case fAttr: - strv := finfo.value(sv) + strv := finfo.value(sv, initNilPointers) if a.Name.Local == finfo.name && (finfo.xmlns == "" || finfo.xmlns == a.Name.Space) { if err := d.unmarshalAttr(strv, a); err != nil { return err @@ -465,7 +465,7 @@ func (d *Decoder) unmarshal(val reflect.Value, start *StartElement) error { } if !handled && any >= 0 { finfo := &tinfo.fields[any] - strv := finfo.value(sv) + strv := finfo.value(sv, initNilPointers) if err := d.unmarshalAttr(strv, a); err != nil { return err } @@ -478,22 +478,22 @@ func (d *Decoder) unmarshal(val reflect.Value, start *StartElement) error { switch finfo.flags & fMode { case fCDATA, fCharData: if !saveData.IsValid() { - saveData = finfo.value(sv) + saveData = finfo.value(sv, initNilPointers) } case fComment: if !saveComment.IsValid() { - saveComment = finfo.value(sv) + saveComment = finfo.value(sv, initNilPointers) } case fAny, fAny | fElement: if !saveAny.IsValid() { - saveAny = finfo.value(sv) + saveAny = finfo.value(sv, initNilPointers) } case fInnerXML: if !saveXML.IsValid() { - saveXML = finfo.value(sv) + saveXML = finfo.value(sv, initNilPointers) if d.saved == nil { saveXMLIndex = 0 d.saved = new(bytes.Buffer) @@ -687,7 +687,7 @@ Loop: } if len(finfo.parents) == len(parents) && finfo.name == start.Name.Local { // It's a perfect match, unmarshal the field. - return true, d.unmarshal(finfo.value(sv), start) + return true, d.unmarshal(finfo.value(sv, initNilPointers), start) } if len(finfo.parents) > len(parents) && finfo.parents[len(parents)] == start.Name.Local { // It's a prefix for the field. Break and recurse diff --git a/libgo/go/encoding/xml/typeinfo.go b/libgo/go/encoding/xml/typeinfo.go index 639952c..f30fe58 100644 --- a/libgo/go/encoding/xml/typeinfo.go +++ b/libgo/go/encoding/xml/typeinfo.go @@ -344,15 +344,25 @@ func (e *TagPathError) Error() string { return fmt.Sprintf("%s field %q with tag %q conflicts with field %q with tag %q", e.Struct, e.Field1, e.Tag1, e.Field2, e.Tag2) } +const ( + initNilPointers = true + dontInitNilPointers = false +) + // value returns v's field value corresponding to finfo. -// It's equivalent to v.FieldByIndex(finfo.idx), but initializes -// and dereferences pointers as necessary. -func (finfo *fieldInfo) value(v reflect.Value) reflect.Value { +// It's equivalent to v.FieldByIndex(finfo.idx), but when passed +// initNilPointers, it initializes and dereferences pointers as necessary. +// When passed dontInitNilPointers and a nil pointer is reached, the function +// returns a zero reflect.Value. +func (finfo *fieldInfo) value(v reflect.Value, shouldInitNilPointers bool) reflect.Value { for i, x := range finfo.idx { if i > 0 { t := v.Type() if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { if v.IsNil() { + if !shouldInitNilPointers { + return reflect.Value{} + } v.Set(reflect.New(v.Type().Elem())) } v = v.Elem() diff --git a/libgo/go/encoding/xml/xml.go b/libgo/go/encoding/xml/xml.go index 5e73dcf..adaf4da 100644 --- a/libgo/go/encoding/xml/xml.go +++ b/libgo/go/encoding/xml/xml.go @@ -960,7 +960,7 @@ func (d *Decoder) ungetc(b byte) { d.offset-- } -var entity = map[string]int{ +var entity = map[string]rune{ "lt": '<', "gt": '>', "amp": '&', @@ -1055,7 +1055,7 @@ Input: d.buf.WriteByte(';') n, err := strconv.ParseUint(s, base, 64) if err == nil && n <= unicode.MaxRune { - text = string(n) + text = string(rune(n)) haveText = true } } |