diff options
Diffstat (limited to 'libgo/go/encoding')
34 files changed, 976 insertions, 324 deletions
diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go index 2ac411a..8bafefd 100644 --- a/libgo/go/encoding/asn1/asn1.go +++ b/libgo/go/encoding/asn1/asn1.go @@ -71,9 +71,28 @@ func parseBool(bytes []byte) (ret bool, err error) { // INTEGER +// checkInteger returns nil if the given bytes are a valid DER-encoded +// INTEGER and an error otherwise. +func checkInteger(bytes []byte) error { + if len(bytes) == 0 { + return StructuralError{"empty integer"} + } + if len(bytes) == 1 { + return nil + } + if (bytes[0] == 0 && bytes[1]&0x80 == 0) || (bytes[0] == 0xff && bytes[1]&0x80 == 0x80) { + return StructuralError{"integer not minimally-encoded"} + } + return nil +} + // parseInt64 treats the given bytes as a big-endian, signed integer and // returns the result. func parseInt64(bytes []byte) (ret int64, err error) { + err = checkInteger(bytes) + if err != nil { + return + } if len(bytes) > 8 { // We'll overflow an int64 in this case. err = StructuralError{"integer too large"} @@ -93,6 +112,9 @@ func parseInt64(bytes []byte) (ret int64, err error) { // parseInt treats the given bytes as a big-endian, signed integer and returns // the result. func parseInt32(bytes []byte) (int32, error) { + if err := checkInteger(bytes); err != nil { + return 0, err + } ret64, err := parseInt64(bytes) if err != nil { return 0, err @@ -107,7 +129,10 @@ var bigOne = big.NewInt(1) // parseBigInt treats the given bytes as a big-endian, signed integer and returns // the result. -func parseBigInt(bytes []byte) *big.Int { +func parseBigInt(bytes []byte) (*big.Int, error) { + if err := checkInteger(bytes); err != nil { + return nil, err + } ret := new(big.Int) if len(bytes) > 0 && bytes[0]&0x80 == 0x80 { // This is a negative number. @@ -118,10 +143,10 @@ func parseBigInt(bytes []byte) *big.Int { ret.SetBytes(notBytes) ret.Add(ret, bigOne) ret.Neg(ret) - return ret + return ret, nil } ret.SetBytes(bytes) - return ret + return ret, nil } // BIT STRING @@ -269,7 +294,7 @@ type Flag bool func parseBase128Int(bytes []byte, initOffset int) (ret, offset int, err error) { offset = initOffset for shifted := 0; offset < len(bytes); shifted++ { - if shifted > 4 { + if shifted == 4 { err = StructuralError{"base 128 integer too large"} return } @@ -475,6 +500,11 @@ func parseTagAndLength(bytes []byte, initOffset int) (ret tagAndLength, offset i return } } + // Short lengths must be encoded in short form. + if ret.length < 0x80 { + err = StructuralError{"non-minimal length"} + return + } } return @@ -500,17 +530,17 @@ func parseSequenceOf(bytes []byte, sliceType reflect.Type, elemType reflect.Type return } switch t.tag { - case tagIA5String, tagGeneralString, tagT61String, tagUTF8String: + case TagIA5String, TagGeneralString, TagT61String, TagUTF8String: // We pretend that various other string types are // PRINTABLE STRINGs so that a sequence of them can be // parsed into a []string. - t.tag = tagPrintableString - case tagGeneralizedTime, tagUTCTime: + t.tag = TagPrintableString + case TagGeneralizedTime, TagUTCTime: // Likewise, both time types are treated the same. - t.tag = tagUTCTime + t.tag = TagUTCTime } - if t.class != classUniversal || t.isCompound != compoundType || t.tag != expectedTag { + if t.class != ClassUniversal || t.isCompound != compoundType || t.tag != expectedTag { err = StructuralError{"sequence tag mismatch"} return } @@ -594,28 +624,28 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam return } var result interface{} - if !t.isCompound && t.class == classUniversal { + if !t.isCompound && t.class == ClassUniversal { innerBytes := bytes[offset : offset+t.length] switch t.tag { - case tagPrintableString: + case TagPrintableString: result, err = parsePrintableString(innerBytes) - case tagIA5String: + case TagIA5String: result, err = parseIA5String(innerBytes) - case tagT61String: + case TagT61String: result, err = parseT61String(innerBytes) - case tagUTF8String: + case TagUTF8String: result, err = parseUTF8String(innerBytes) - case tagInteger: + case TagInteger: result, err = parseInt64(innerBytes) - case tagBitString: + case TagBitString: result, err = parseBitString(innerBytes) - case tagOID: + case TagOID: result, err = parseObjectIdentifier(innerBytes) - case tagUTCTime: + case TagUTCTime: result, err = parseUTCTime(innerBytes) - case tagGeneralizedTime: + case TagGeneralizedTime: result, err = parseGeneralizedTime(innerBytes) - case tagOctetString: + case TagOctetString: result = innerBytes default: // If we don't know how to handle the type, we just leave Value as nil. @@ -641,9 +671,9 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam return } if params.explicit { - expectedClass := classContextSpecific + expectedClass := ClassContextSpecific if params.application { - expectedClass = classApplication + expectedClass = ClassApplication } if offset == len(bytes) { err = StructuralError{"explicit tag has no child"} @@ -679,10 +709,10 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam // type string. getUniversalType returns the tag for PrintableString // when it sees a string, so if we see a different string type on the // wire, we change the universal type to match. - if universalTag == tagPrintableString { - if t.class == classUniversal { + if universalTag == TagPrintableString { + if t.class == ClassUniversal { switch t.tag { - case tagIA5String, tagGeneralString, tagT61String, tagUTF8String: + case TagIA5String, TagGeneralString, TagT61String, TagUTF8String: universalTag = t.tag } } else if params.stringType != 0 { @@ -692,24 +722,24 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam // Special case for time: UTCTime and GeneralizedTime both map to the // Go type time.Time. - if universalTag == tagUTCTime && t.tag == tagGeneralizedTime && t.class == classUniversal { - universalTag = tagGeneralizedTime + if universalTag == TagUTCTime && t.tag == TagGeneralizedTime && t.class == ClassUniversal { + universalTag = TagGeneralizedTime } if params.set { - universalTag = tagSet + universalTag = TagSet } - expectedClass := classUniversal + expectedClass := ClassUniversal expectedTag := universalTag if !params.explicit && params.tag != nil { - expectedClass = classContextSpecific + expectedClass = ClassContextSpecific expectedTag = *params.tag } if !params.explicit && params.application && params.tag != nil { - expectedClass = classApplication + expectedClass = ClassApplication expectedTag = *params.tag } @@ -751,7 +781,7 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam case timeType: var time time.Time var err1 error - if universalTag == tagUTCTime { + if universalTag == TagUTCTime { time, err1 = parseUTCTime(innerBytes) } else { time, err1 = parseGeneralizedTime(innerBytes) @@ -772,8 +802,11 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam v.SetBool(true) return case bigIntType: - parsedInt := parseBigInt(innerBytes) - v.Set(reflect.ValueOf(parsedInt)) + parsedInt, err1 := parseBigInt(innerBytes) + if err1 == nil { + v.Set(reflect.ValueOf(parsedInt)) + } + err = err1 return } switch val := v; val.Kind() { @@ -840,15 +873,15 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam case reflect.String: var v string switch universalTag { - case tagPrintableString: + case TagPrintableString: v, err = parsePrintableString(innerBytes) - case tagIA5String: + case TagIA5String: v, err = parseIA5String(innerBytes) - case tagT61String: + case TagT61String: v, err = parseT61String(innerBytes) - case tagUTF8String: + case TagUTF8String: v, err = parseUTF8String(innerBytes) - case tagGeneralString: + case TagGeneralString: // GeneralString is specified in ISO-2022/ECMA-35, // A brief review suggests that it includes structures // that allow the encoding to change midstring and diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go index 893d080..e0e8331 100644 --- a/libgo/go/encoding/asn1/asn1_test.go +++ b/libgo/go/encoding/asn1/asn1_test.go @@ -53,10 +53,12 @@ var int64TestData = []int64Test{ {[]byte{0x01, 0x00}, true, 256}, {[]byte{0x80}, true, -128}, {[]byte{0xff, 0x7f}, true, -129}, - {[]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, true, -1}, {[]byte{0xff}, true, -1}, {[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, true, -9223372036854775808}, {[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, false, 0}, + {[]byte{}, false, 0}, + {[]byte{0x00, 0x7f}, false, 0}, + {[]byte{0xff, 0xf0}, false, 0}, } func TestParseInt64(t *testing.T) { @@ -84,10 +86,12 @@ var int32TestData = []int32Test{ {[]byte{0x01, 0x00}, true, 256}, {[]byte{0x80}, true, -128}, {[]byte{0xff, 0x7f}, true, -129}, - {[]byte{0xff, 0xff, 0xff, 0xff}, true, -1}, {[]byte{0xff}, true, -1}, {[]byte{0x80, 0x00, 0x00, 0x00}, true, -2147483648}, {[]byte{0x80, 0x00, 0x00, 0x00, 0x00}, false, 0}, + {[]byte{}, false, 0}, + {[]byte{0x00, 0x7f}, false, 0}, + {[]byte{0xff, 0xf0}, false, 0}, } func TestParseInt32(t *testing.T) { @@ -104,27 +108,36 @@ func TestParseInt32(t *testing.T) { var bigIntTests = []struct { in []byte + ok bool base10 string }{ - {[]byte{0xff}, "-1"}, - {[]byte{0x00}, "0"}, - {[]byte{0x01}, "1"}, - {[]byte{0x00, 0xff}, "255"}, - {[]byte{0xff, 0x00}, "-256"}, - {[]byte{0x01, 0x00}, "256"}, + {[]byte{0xff}, true, "-1"}, + {[]byte{0x00}, true, "0"}, + {[]byte{0x01}, true, "1"}, + {[]byte{0x00, 0xff}, true, "255"}, + {[]byte{0xff, 0x00}, true, "-256"}, + {[]byte{0x01, 0x00}, true, "256"}, + {[]byte{}, false, ""}, + {[]byte{0x00, 0x7f}, false, ""}, + {[]byte{0xff, 0xf0}, false, ""}, } func TestParseBigInt(t *testing.T) { for i, test := range bigIntTests { - ret := parseBigInt(test.in) - if ret.String() != test.base10 { - t.Errorf("#%d: bad result from %x, got %s want %s", i, test.in, ret.String(), test.base10) + ret, err := parseBigInt(test.in) + if (err == nil) != test.ok { + t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) } - fw := newForkableWriter() - marshalBigInt(fw, ret) - result := fw.Bytes() - if !bytes.Equal(result, test.in) { - t.Errorf("#%d: got %x from marshaling %s, want %x", i, result, ret, test.in) + if test.ok { + if ret.String() != test.base10 { + t.Errorf("#%d: bad result from %x, got %s want %s", i, test.in, ret.String(), test.base10) + } + fw := newForkableWriter() + marshalBigInt(fw, ret) + result := fw.Bytes() + if !bytes.Equal(result, test.in) { + t.Errorf("#%d: got %x from marshaling %s, want %x", i, result, ret, test.in) + } } } } @@ -354,17 +367,21 @@ var tagAndLengthData = []tagAndLengthTest{ {[]byte{0x1f, 0x01, 0x00}, true, tagAndLength{0, 1, 0, false}}, {[]byte{0x1f, 0x81, 0x00, 0x00}, true, tagAndLength{0, 128, 0, false}}, {[]byte{0x1f, 0x81, 0x80, 0x01, 0x00}, true, tagAndLength{0, 0x4001, 0, false}}, - {[]byte{0x00, 0x81, 0x01}, true, tagAndLength{0, 0, 1, false}}, + {[]byte{0x00, 0x81, 0x80}, true, tagAndLength{0, 0, 128, false}}, {[]byte{0x00, 0x82, 0x01, 0x00}, true, tagAndLength{0, 0, 256, false}}, {[]byte{0x00, 0x83, 0x01, 0x00}, false, tagAndLength{}}, {[]byte{0x1f, 0x85}, false, tagAndLength{}}, {[]byte{0x30, 0x80}, false, tagAndLength{}}, // Superfluous zeros in the length should be an error. - {[]byte{0xa0, 0x82, 0x00, 0x01}, false, tagAndLength{}}, + {[]byte{0xa0, 0x82, 0x00, 0xff}, false, tagAndLength{}}, // Lengths up to the maximum size of an int should work. {[]byte{0xa0, 0x84, 0x7f, 0xff, 0xff, 0xff}, true, tagAndLength{2, 0, 0x7fffffff, true}}, // Lengths that would overflow an int should be rejected. {[]byte{0xa0, 0x84, 0x80, 0x00, 0x00, 0x00}, false, tagAndLength{}}, + // Long length form may not be used for lengths that fit in short form. + {[]byte{0xa0, 0x81, 0x7f}, false, tagAndLength{}}, + // Tag numbers which would overflow int32 are rejected. (The value below is 2^31.) + {[]byte{0x1f, 0x88, 0x80, 0x80, 0x80, 0x00, 0x00}, false, tagAndLength{}}, } func TestParseTagAndLength(t *testing.T) { @@ -394,10 +411,10 @@ func newBool(b bool) *bool { return &b } var parseFieldParametersTestData []parseFieldParametersTest = []parseFieldParametersTest{ {"", fieldParameters{}}, - {"ia5", fieldParameters{stringType: tagIA5String}}, - {"generalized", fieldParameters{timeType: tagGeneralizedTime}}, - {"utc", fieldParameters{timeType: tagUTCTime}}, - {"printable", fieldParameters{stringType: tagPrintableString}}, + {"ia5", fieldParameters{stringType: TagIA5String}}, + {"generalized", fieldParameters{timeType: TagGeneralizedTime}}, + {"utc", fieldParameters{timeType: TagUTCTime}}, + {"printable", fieldParameters{stringType: TagPrintableString}}, {"optional", fieldParameters{optional: true}}, {"explicit", fieldParameters{explicit: true, tag: new(int)}}, {"application", fieldParameters{application: true, tag: new(int)}}, @@ -940,3 +957,15 @@ func TestUnmarshalInvalidUTF8(t *testing.T) { t.Fatalf("Expected error to mention %q but error was %q", expectedSubstring, err.Error()) } } + +func TestMarshalNilValue(t *testing.T) { + nilValueTestData := []interface{}{ + nil, + struct{ v interface{} }{}, + } + for i, test := range nilValueTestData { + if _, err := Marshal(test); err == nil { + t.Fatalf("#%d: successfully marshaled nil value", i) + } + } +} diff --git a/libgo/go/encoding/asn1/common.go b/libgo/go/encoding/asn1/common.go index ab85e04..0695180 100644 --- a/libgo/go/encoding/asn1/common.go +++ b/libgo/go/encoding/asn1/common.go @@ -18,29 +18,31 @@ import ( // Here are some standard tags and classes +// ASN.1 tags represent the type of the following object. const ( - tagBoolean = 1 - tagInteger = 2 - tagBitString = 3 - tagOctetString = 4 - tagOID = 6 - tagEnum = 10 - tagUTF8String = 12 - tagSequence = 16 - tagSet = 17 - tagPrintableString = 19 - tagT61String = 20 - tagIA5String = 22 - tagUTCTime = 23 - tagGeneralizedTime = 24 - tagGeneralString = 27 + TagBoolean = 1 + TagInteger = 2 + TagBitString = 3 + TagOctetString = 4 + TagOID = 6 + TagEnum = 10 + TagUTF8String = 12 + TagSequence = 16 + TagSet = 17 + TagPrintableString = 19 + TagT61String = 20 + TagIA5String = 22 + TagUTCTime = 23 + TagGeneralizedTime = 24 + TagGeneralString = 27 ) +// ASN.1 class types represent the namespace of the tag. const ( - classUniversal = 0 - classApplication = 1 - classContextSpecific = 2 - classPrivate = 3 + ClassUniversal = 0 + ClassApplication = 1 + ClassContextSpecific = 2 + ClassPrivate = 3 ) type tagAndLength struct { @@ -96,15 +98,15 @@ func parseFieldParameters(str string) (ret fieldParameters) { ret.tag = new(int) } case part == "generalized": - ret.timeType = tagGeneralizedTime + ret.timeType = TagGeneralizedTime case part == "utc": - ret.timeType = tagUTCTime + ret.timeType = TagUTCTime case part == "ia5": - ret.stringType = tagIA5String + ret.stringType = TagIA5String case part == "printable": - ret.stringType = tagPrintableString + ret.stringType = TagPrintableString case part == "utf8": - ret.stringType = tagUTF8String + ret.stringType = TagUTF8String case strings.HasPrefix(part, "default:"): i, err := strconv.ParseInt(part[8:], 10, 64) if err == nil { @@ -136,33 +138,33 @@ func parseFieldParameters(str string) (ret fieldParameters) { func getUniversalType(t reflect.Type) (tagNumber int, isCompound, ok bool) { switch t { case objectIdentifierType: - return tagOID, false, true + return TagOID, false, true case bitStringType: - return tagBitString, false, true + return TagBitString, false, true case timeType: - return tagUTCTime, false, true + return TagUTCTime, false, true case enumeratedType: - return tagEnum, false, true + return TagEnum, false, true case bigIntType: - return tagInteger, false, true + return TagInteger, false, true } switch t.Kind() { case reflect.Bool: - return tagBoolean, false, true + return TagBoolean, false, true case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return tagInteger, false, true + return TagInteger, false, true case reflect.Struct: - return tagSequence, true, true + return TagSequence, true, true case reflect.Slice: if t.Elem().Kind() == reflect.Uint8 { - return tagOctetString, false, true + return TagOctetString, false, true } if strings.HasSuffix(t.Name(), "SET") { - return tagSet, true, true + return TagSet, true, true } - return tagSequence, true, true + return TagSequence, true, true case reflect.String: - return tagPrintableString, false, true + return TagPrintableString, false, true } return 0, false, false } diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index 67a019d..6e85858 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -414,7 +414,7 @@ func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameter return nil case timeType: t := value.Interface().(time.Time) - if params.timeType == tagGeneralizedTime || outsideUTCRange(t) { + if params.timeType == TagGeneralizedTime || outsideUTCRange(t) { return marshalGeneralizedTime(out, t) } else { return marshalUTCTime(out, t) @@ -493,9 +493,9 @@ func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameter return case reflect.String: switch params.stringType { - case tagIA5String: + case TagIA5String: return marshalIA5String(out, v.String()) - case tagPrintableString: + case TagPrintableString: return marshalPrintableString(out, v.String()) default: return marshalUTF8String(out, v.String()) @@ -506,6 +506,9 @@ func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameter } func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) (err error) { + if !v.IsValid() { + return fmt.Errorf("asn1: cannot marshal nil value") + } // If the field is an interface{} then recurse into it. if v.Kind() == reflect.Interface && v.Type().NumMethod() == 0 { return marshalField(out, v.Elem(), params) @@ -552,18 +555,18 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) err = StructuralError{fmt.Sprintf("unknown Go type: %v", v.Type())} return } - class := classUniversal + class := ClassUniversal - if params.timeType != 0 && tag != tagUTCTime { + if params.timeType != 0 && tag != TagUTCTime { return StructuralError{"explicit time type given to non-time member"} } - if params.stringType != 0 && tag != tagPrintableString { + if params.stringType != 0 && tag != TagPrintableString { return StructuralError{"explicit string type given to non-string member"} } switch tag { - case tagPrintableString: + case TagPrintableString: if params.stringType == 0 { // This is a string without an explicit string type. We'll use // a PrintableString if the character set in the string is @@ -573,24 +576,24 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) if !utf8.ValidString(v.String()) { return errors.New("asn1: string not valid UTF-8") } - tag = tagUTF8String + tag = TagUTF8String break } } } else { tag = params.stringType } - case tagUTCTime: - if params.timeType == tagGeneralizedTime || outsideUTCRange(v.Interface().(time.Time)) { - tag = tagGeneralizedTime + case TagUTCTime: + if params.timeType == TagGeneralizedTime || outsideUTCRange(v.Interface().(time.Time)) { + tag = TagGeneralizedTime } } if params.set { - if tag != tagSequence { + if tag != TagSequence { return StructuralError{"non sequence tagged as set"} } - tag = tagSet + tag = TagSet } tags, body := out.fork() @@ -610,7 +613,7 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) if !params.explicit && params.tag != nil { // implicit tag. tag = *params.tag - class = classContextSpecific + class = ClassContextSpecific } err = marshalTagAndLength(tags, tagAndLength{class, tag, bodyLen, isCompound}) @@ -620,14 +623,14 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) if params.explicit { err = marshalTagAndLength(explicitTag, tagAndLength{ - class: classContextSpecific, + class: ClassContextSpecific, tag: *params.tag, length: bodyLen + tags.Len(), isCompound: true, }) } - return nil + return err } // Marshal returns the ASN.1 encoding of val. @@ -648,5 +651,5 @@ func Marshal(val interface{}) ([]byte, error) { return nil, err } _, err = f.writeTo(&out) - return out.Bytes(), nil + return out.Bytes(), err } diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go index 3302fb4..1bda804 100644 --- a/libgo/go/encoding/base64/base64.go +++ b/libgo/go/encoding/base64/base64.go @@ -75,7 +75,7 @@ var URLEncoding = NewEncoding(encodeURL) // This is the same as StdEncoding but omits padding characters. var RawStdEncoding = StdEncoding.WithPadding(NoPadding) -// URLEncoding is the unpadded alternate base64 encoding defined in RFC 4648. +// RawURLEncoding is the unpadded alternate base64 encoding defined in RFC 4648. // It is typically used in URLs and file names. // This is the same as URLEncoding but omits padding characters. var RawURLEncoding = URLEncoding.WithPadding(NoPadding) @@ -346,21 +346,18 @@ func (enc *Encoding) DecodeString(s string) ([]byte, error) { } type decoder struct { - err error - enc *Encoding - r io.Reader - end bool // saw end of message - buf [1024]byte // leftover input - nbuf int - out []byte // leftover decoded output - outbuf [1024 / 4 * 3]byte + err error + readErr error // error from r.Read + enc *Encoding + r io.Reader + end bool // saw end of message + buf [1024]byte // leftover input + nbuf int + out []byte // leftover decoded output + outbuf [1024 / 4 * 3]byte } func (d *decoder) Read(p []byte) (n int, err error) { - if d.err != nil { - return 0, d.err - } - // Use leftover decoded output from last read. if len(d.out) > 0 { n = copy(p, d.out) @@ -368,19 +365,46 @@ func (d *decoder) Read(p []byte) (n int, err error) { return n, nil } + if d.err != nil { + return 0, d.err + } + // This code assumes that d.r strips supported whitespace ('\r' and '\n'). - // Read a chunk. - nn := len(p) / 3 * 4 - if nn < 4 { - nn = 4 - } - if nn > len(d.buf) { - nn = len(d.buf) + // Refill buffer. + for d.nbuf < 4 && d.readErr == nil { + nn := len(p) / 3 * 4 + if nn < 4 { + nn = 4 + } + if nn > len(d.buf) { + nn = len(d.buf) + } + nn, d.readErr = d.r.Read(d.buf[d.nbuf:nn]) + d.nbuf += nn } - nn, d.err = io.ReadAtLeast(d.r, d.buf[d.nbuf:nn], 4-d.nbuf) - d.nbuf += nn - if d.err != nil || d.nbuf < 4 { + + if d.nbuf < 4 { + if d.enc.padChar == NoPadding && d.nbuf > 0 { + // Decode final fragment, without padding. + var nw int + nw, _, d.err = d.enc.decode(d.outbuf[:], d.buf[:d.nbuf]) + d.nbuf = 0 + d.end = true + d.out = d.outbuf[:nw] + n = copy(p, d.out) + d.out = d.out[n:] + if n > 0 || len(p) == 0 && len(d.out) > 0 { + return n, nil + } + if d.err != nil { + return 0, d.err + } + } + d.err = d.readErr + if d.err == io.EOF && d.nbuf > 0 { + d.err = io.ErrUnexpectedEOF + } return 0, d.err } @@ -396,13 +420,7 @@ func (d *decoder) Read(p []byte) (n int, err error) { n, d.end, d.err = d.enc.decode(p, d.buf[:nr]) } d.nbuf -= nr - for i := 0; i < d.nbuf; i++ { - d.buf[i] = d.buf[i+nr] - } - - if d.err == nil { - d.err = err - } + copy(d.buf[:d.nbuf], d.buf[nr:]) return n, d.err } diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go index d144b96..fc6a1ea 100644 --- a/libgo/go/encoding/base64/base64_test.go +++ b/libgo/go/encoding/base64/base64_test.go @@ -80,11 +80,11 @@ type encodingTest struct { } var encodingTests = []encodingTest{ - encodingTest{StdEncoding, stdRef}, - encodingTest{URLEncoding, urlRef}, - encodingTest{RawStdEncoding, rawRef}, - encodingTest{RawURLEncoding, rawUrlRef}, - encodingTest{funnyEncoding, funnyRef}, + {StdEncoding, stdRef}, + {URLEncoding, urlRef}, + {RawStdEncoding, rawRef}, + {RawURLEncoding, rawUrlRef}, + {funnyEncoding, funnyRef}, } var bigtest = testpair{ @@ -406,3 +406,28 @@ func BenchmarkDecodeString(b *testing.B) { StdEncoding.DecodeString(data) } } + +func TestDecoderRaw(t *testing.T) { + source := "AAAAAA" + want := []byte{0, 0, 0, 0} + + // Direct. + dec1, err := RawURLEncoding.DecodeString(source) + if err != nil || !bytes.Equal(dec1, want) { + t.Errorf("RawURLEncoding.DecodeString(%q) = %x, %v, want %x, nil", source, dec1, err, want) + } + + // Through reader. Used to fail. + r := NewDecoder(RawURLEncoding, bytes.NewReader([]byte(source))) + dec2, err := ioutil.ReadAll(io.LimitReader(r, 100)) + if err != nil || !bytes.Equal(dec2, want) { + t.Errorf("reading NewDecoder(RawURLEncoding, %q) = %x, %v, want %x, nil", source, dec2, err, want) + } + + // Should work with padding. + r = NewDecoder(URLEncoding, bytes.NewReader([]byte(source+"=="))) + dec3, err := ioutil.ReadAll(r) + if err != nil || !bytes.Equal(dec3, want) { + t.Errorf("reading NewDecoder(URLEncoding, %q) = %x, %v, want %x, nil", source+"==", dec3, err, want) + } +} diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index 2bbe07c..1c2577b 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -135,6 +135,10 @@ func (bigEndian) GoString() string { return "binary.BigEndian" } // blank (_) field names is skipped; i.e., blank field names // may be used for padding. // When reading into a struct, all non-blank fields must be exported. +// +// The error is EOF only if no bytes were read. +// If an EOF happens after reading some but not all the bytes, +// Read returns ErrUnexpectedEOF. func Read(r io.Reader, order ByteOrder, data interface{}) error { // Fast path for basic types and slices. if n := intDataSize(data); n != 0 { diff --git a/libgo/go/encoding/binary/binary_test.go b/libgo/go/encoding/binary/binary_test.go index 8ee595f..7fd36fa 100644 --- a/libgo/go/encoding/binary/binary_test.go +++ b/libgo/go/encoding/binary/binary_test.go @@ -309,6 +309,36 @@ func TestReadErrorMsg(t *testing.T) { read(&p) } +func TestReadTruncated(t *testing.T) { + const data = "0123456789abcdef" + + var b1 = make([]int32, 4) + var b2 struct { + A, B, C, D byte + E int32 + F float64 + } + + for i := 0; i <= len(data); i++ { + var errWant error + switch i { + case 0: + errWant = io.EOF + case len(data): + errWant = nil + default: + errWant = io.ErrUnexpectedEOF + } + + if err := Read(strings.NewReader(data[:i]), LittleEndian, &b1); err != errWant { + t.Errorf("Read(%d) with slice: got %v, want %v", i, err, errWant) + } + if err := Read(strings.NewReader(data[:i]), LittleEndian, &b2); err != errWant { + t.Errorf("Read(%d) with struct: got %v, want %v", i, err, errWant) + } + } +} + type byteSliceReader struct { remain []byte } diff --git a/libgo/go/encoding/csv/reader.go b/libgo/go/encoding/csv/reader.go index 37bf80c..a6bb780 100644 --- a/libgo/go/encoding/csv/reader.go +++ b/libgo/go/encoding/csv/reader.go @@ -155,7 +155,7 @@ func (r *Reader) Read() (record []string, err error) { // ReadAll reads all the remaining records from r. // Each record is a slice of fields. -// A successful call returns err == nil, not err == EOF. Because ReadAll is +// A successful call returns err == nil, not err == io.EOF. Because ReadAll is // defined to read until EOF, it does not treat end of file as an error to be // reported. func (r *Reader) ReadAll() (records [][]string, err error) { diff --git a/libgo/go/encoding/gob/codec_test.go b/libgo/go/encoding/gob/codec_test.go index c2583bf..8efcdc7 100644 --- a/libgo/go/encoding/gob/codec_test.go +++ b/libgo/go/encoding/gob/codec_test.go @@ -88,7 +88,6 @@ func verifyInt(i int64, t *testing.T) { encState := newEncoderState(b) encState.encodeInt(i) decState := newDecodeState(newDecBuffer(b.Bytes())) - decState.buf = make([]byte, 8) j := decState.decodeInt() if i != j { t.Errorf("Encode/Decode: sent %#x received %#x", uint64(i), uint64(j)) @@ -127,7 +126,6 @@ var bytesResult = []byte{0x07, 0x05, 'h', 'e', 'l', 'l', 'o'} func newDecodeState(buf *decBuffer) *decoderState { d := new(decoderState) d.b = buf - d.buf = make([]byte, uint64Size) return d } @@ -1488,7 +1486,7 @@ func TestErrorInvalidTypeId(t *testing.T) { var foo struct{} err := d.Decode(&foo) if err != errBadType { - t.Fatal("decode: expected %s, got %s", errBadType, err) + t.Fatalf("decode: expected %s, got %s", errBadType, err) } } } diff --git a/libgo/go/encoding/gob/dec_helpers.go b/libgo/go/encoding/gob/dec_helpers.go index a1b6766..3aa038d 100644 --- a/libgo/go/encoding/gob/dec_helpers.go +++ b/libgo/go/encoding/gob/dec_helpers.go @@ -327,11 +327,12 @@ func decStringSlice(state *decoderState, v reflect.Value, length int, ovfl error errorf("string data too long for buffer: %d", n) } // Read the data. - data := make([]byte, n) - if _, err := state.b.Read(data); err != nil { - errorf("error decoding string: %s", err) + data := state.b.Bytes() + if len(data) < n { + errorf("invalid string length %d: exceeds input size %d", n, len(data)) } - slice[i] = string(data) + slice[i] = string(data[:n]) + state.b.Drop(n) } return true } diff --git a/libgo/go/encoding/gob/decgen.go b/libgo/go/encoding/gob/decgen.go index da41a89..ef73f2d 100644 --- a/libgo/go/encoding/gob/decgen.go +++ b/libgo/go/encoding/gob/decgen.go @@ -112,11 +112,12 @@ var types = []Type{ errorf("string data too long for buffer: %d", n) } // Read the data. - data := make([]byte, n) - if _, err := state.b.Read(data); err != nil { - errorf("error decoding string: %s", err) + data := state.b.Bytes() + if len(data) < n { + errorf("invalid string length %d: exceeds input size %d", n, len(data)) } - slice[i] = string(data)`, + slice[i] = string(data[:n]) + state.b.Drop(n)`, }, { "uint", diff --git a/libgo/go/encoding/gob/decode.go b/libgo/go/encoding/gob/decode.go index e913f15..3b0dca8 100644 --- a/libgo/go/encoding/gob/decode.go +++ b/libgo/go/encoding/gob/decode.go @@ -29,8 +29,7 @@ type decoderState struct { // The buffer is stored with an extra indirection because it may be replaced // if we load a type during decode (when reading an interface value). b *decBuffer - fieldnum int // the last field number read. - buf []byte + fieldnum int // the last field number read. next *decoderState // for free list } @@ -97,7 +96,6 @@ func (dec *Decoder) newDecoderState(buf *decBuffer) *decoderState { if d == nil { d = new(decoderState) d.dec = dec - d.buf = make([]byte, uint64Size) } else { dec.freeList = d.next } @@ -160,15 +158,16 @@ func (state *decoderState) decodeUint() (x uint64) { if n > uint64Size { error_(errBadUint) } - width, err := state.b.Read(state.buf[0:n]) - if err != nil { - error_(err) + buf := state.b.Bytes() + if len(buf) < n { + errorf("invalid uint data length %d: exceeds input size %d", n, len(buf)) } // Don't need to check error; it's safe to loop regardless. // Could check that the high byte is zero but it's not worth it. - for _, b := range state.buf[0:width] { + for _, b := range buf[0:n] { x = x<<8 | uint64(b) } + state.b.Drop(n) return x } @@ -397,11 +396,13 @@ func decString(i *decInstr, state *decoderState, value reflect.Value) { errorf("bad %s slice length: %d", value.Type(), n) } // Read the data. - data := make([]byte, n) - if _, err := state.b.Read(data); err != nil { - errorf("error decoding string: %s", err) + data := state.b.Bytes() + if len(data) < n { + errorf("invalid string length %d: exceeds input size %d", n, len(data)) } - value.SetString(string(data)) + s := string(data[:n]) + state.b.Drop(n) + value.SetString(s) } // ignoreUint8Array skips over the data for a byte slice value with no destination. @@ -410,8 +411,11 @@ func ignoreUint8Array(i *decInstr, state *decoderState, value reflect.Value) { if !ok { errorf("slice length too large") } - b := make([]byte, n) - state.b.Read(b) + bn := state.b.Len() + if bn < n { + errorf("invalid slice length %d: exceeds input size %d", n, bn) + } + state.b.Drop(n) } // Execution engine @@ -634,15 +638,15 @@ func (dec *Decoder) ignoreSlice(state *decoderState, elemOp decOp) { func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, value reflect.Value) { // Read the name of the concrete type. nr := state.decodeUint() - if nr < 0 || nr > 1<<31 { // zero is permissible for anonymous types + if nr > 1<<31 { // zero is permissible for anonymous types errorf("invalid type name length %d", nr) } if nr > uint64(state.b.Len()) { errorf("invalid type name length %d: exceeds input size", nr) } - b := make([]byte, nr) - state.b.Read(b) - name := string(b) + n := int(nr) + name := string(state.b.Bytes()[:n]) + state.b.Drop(n) // Allocate the destination interface value. if name == "" { // Copy the nil interface value to the target. @@ -689,11 +693,11 @@ func (dec *Decoder) ignoreInterface(state *decoderState) { if !ok { errorf("bad interface encoding: name too large for buffer") } - b := make([]byte, n) - _, err := state.b.Read(b) - if err != nil { - error_(err) + bn := state.b.Len() + if bn < n { + errorf("invalid interface value length %d: exceeds input size %d", n, bn) } + state.b.Drop(n) id := dec.decodeTypeSequence(true) if id < 0 { error_(dec.err) @@ -714,11 +718,13 @@ func (dec *Decoder) decodeGobDecoder(ut *userTypeInfo, state *decoderState, valu if !ok { errorf("GobDecoder: length too large for buffer") } - b := make([]byte, n) - _, err := state.b.Read(b) - if err != nil { - error_(err) + b := state.b.Bytes() + if len(b) < n { + errorf("GobDecoder: invalid data length %d: exceeds input size %d", n, len(b)) } + b = b[:n] + state.b.Drop(n) + var err error // We know it's one of these. switch ut.externalDec { case xGob: @@ -740,11 +746,11 @@ func (dec *Decoder) ignoreGobDecoder(state *decoderState) { if !ok { errorf("GobDecoder: length too large for buffer") } - b := make([]byte, n) - _, err := state.b.Read(b) - if err != nil { - error_(err) + bn := state.b.Len() + if bn < n { + errorf("GobDecoder: invalid data length %d: exceeds input size %d", n, bn) } + state.b.Drop(n) } // Index by Go types. diff --git a/libgo/go/encoding/gob/doc.go b/libgo/go/encoding/gob/doc.go index 4d3d007..cf878f4 100644 --- a/libgo/go/encoding/gob/doc.go +++ b/libgo/go/encoding/gob/doc.go @@ -82,6 +82,12 @@ slice has capacity the slice will be extended in place; if not, a new array is allocated. Regardless, the length of the resulting slice reports the number of elements decoded. +In general, if allocation is required, the decoder will allocate memory. If not, +it will update the destination variables with values read from the stream. It does +not initialize them first, so if the destination is a compound value such as a +map, struct, or slice, the decoded values will be merged elementwise into the +existing variables. + Functions and channels will not be sent in a gob. Attempting to encode such a value at the top level will fail. A struct field of chan or func type is treated exactly like an unexported field and is ignored. @@ -141,18 +147,21 @@ pairs. Empty but non-nil maps are sent, so if the receiver has not allocated one already, one will always be allocated on receipt unless the transmitted map is nil and not at the top level. +In slices and arrays, as well as maps, all elements, even zero-valued elements, +are transmitted, even if all the elements are zero. + Structs are sent as a sequence of (field number, field value) pairs. The field value is sent using the standard gob encoding for its type, recursively. If a -field has the zero value for its type, it is omitted from the transmission. The -field number is defined by the type of the encoded struct: the first field of the -encoded type is field 0, the second is field 1, etc. When encoding a value, the -field numbers are delta encoded for efficiency and the fields are always sent in -order of increasing field number; the deltas are therefore unsigned. The -initialization for the delta encoding sets the field number to -1, so an unsigned -integer field 0 with value 7 is transmitted as unsigned delta = 1, unsigned value -= 7 or (01 07). Finally, after all the fields have been sent a terminating mark -denotes the end of the struct. That mark is a delta=0 value, which has -representation (00). +field has the zero value for its type (except for arrays; see above), it is omitted +from the transmission. The field number is defined by the type of the encoded +struct: the first field of the encoded type is field 0, the second is field 1, +etc. When encoding a value, the field numbers are delta encoded for efficiency +and the fields are always sent in order of increasing field number; the deltas are +therefore unsigned. The initialization for the delta encoding sets the field +number to -1, so an unsigned integer field 0 with value 7 is transmitted as unsigned +delta = 1, unsigned value = 7 or (01 07). Finally, after all the fields have been +sent a terminating mark denotes the end of the struct. That mark is a delta=0 +value, which has representation (00). Interface types are not checked for compatibility; all interface types are treated, for transmission, as members of a single "interface" type, analogous to diff --git a/libgo/go/encoding/gob/encode.go b/libgo/go/encoding/gob/encode.go index f66279f..96052ef 100644 --- a/libgo/go/encoding/gob/encode.go +++ b/libgo/go/encoding/gob/encode.go @@ -10,6 +10,7 @@ import ( "encoding" "math" "reflect" + "sync" ) const uint64Size = 8 @@ -36,6 +37,14 @@ type encBuffer struct { scratch [64]byte } +var encBufferPool = sync.Pool{ + New: func() interface{} { + e := new(encBuffer) + e.data = e.scratch[0:0] + return e + }, +} + func (e *encBuffer) WriteByte(c byte) { e.data = append(e.data, c) } @@ -58,7 +67,11 @@ func (e *encBuffer) Bytes() []byte { } func (e *encBuffer) Reset() { - e.data = e.data[0:0] + if len(e.data) >= tooBig { + e.data = e.scratch[0:0] + } else { + e.data = e.data[0:0] + } } func (enc *Encoder) newEncoderState(b *encBuffer) *encoderState { @@ -407,7 +420,7 @@ func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) { // Encode the value into a new buffer. Any nested type definitions // should be written to b, before the encoded value. enc.pushWriter(b) - data := new(encBuffer) + data := encBufferPool.Get().(*encBuffer) data.Write(spaceForLength) enc.encode(data, elem, ut) if enc.err != nil { @@ -415,6 +428,8 @@ func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) { } enc.popWriter() enc.writeMessage(b, data) + data.Reset() + encBufferPool.Put(data) if enc.err != nil { error_(enc.err) } diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index dc65734..570d796 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -978,7 +978,7 @@ var badDataTests = []badDataTest{ {"0f1000fb285d003316020735ff023a65c5", "interface encoding", nil}, {"03fffb0616fffc00f902ff02ff03bf005d02885802a311a8120228022c028ee7", "GobDecoder", nil}, // Issue 10491. - {"10fe010f020102fe01100001fe010e000016fe010d030102fe010e00010101015801fe01100000000bfe011000f85555555555555555", "length exceeds input size", nil}, + {"10fe010f020102fe01100001fe010e000016fe010d030102fe010e00010101015801fe01100000000bfe011000f85555555555555555", "exceeds input size", nil}, } // TestBadData tests that various problems caused by malformed input diff --git a/libgo/go/encoding/gob/timing_test.go b/libgo/go/encoding/gob/timing_test.go index 940e5ad..424b7e6 100644 --- a/libgo/go/encoding/gob/timing_test.go +++ b/libgo/go/encoding/gob/timing_test.go @@ -127,8 +127,8 @@ func TestCountDecodeMallocs(t *testing.T) { t.Fatal("decode:", err) } }) - if allocs != 4 { - t.Fatalf("mallocs per decode of type Bench: %v; wanted 4\n", allocs) + if allocs != 3 { + t.Fatalf("mallocs per decode of type Bench: %v; wanted 3\n", allocs) } } @@ -200,6 +200,23 @@ func BenchmarkEncodeStringSlice(b *testing.B) { } } +func BenchmarkEncodeInterfaceSlice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]interface{}, 1000) + for i := range a { + a[i] = "now is the time" + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf.Reset() + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + } +} + // benchmarkBuf is a read buffer we can reset type benchmarkBuf struct { offset int @@ -323,3 +340,27 @@ func BenchmarkDecodeStringSlice(b *testing.B) { } } } + +func BenchmarkDecodeInterfaceSlice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]interface{}, 1000) + for i := range a { + a[i] = "now is the time" + } + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + x := make([]interface{}, 1000) + bbuf := benchmarkBuf{data: buf.Bytes()} + b.ResetTimer() + for i := 0; i < b.N; i++ { + bbuf.reset() + dec := NewDecoder(&bbuf) + err := dec.Decode(&x) + if err != nil { + b.Fatal(i, err) + } + } +} diff --git a/libgo/go/encoding/gob/type.go b/libgo/go/encoding/gob/type.go index a49b71a..cf5cec0 100644 --- a/libgo/go/encoding/gob/type.go +++ b/libgo/go/encoding/gob/type.go @@ -787,7 +787,7 @@ func mustGetTypeInfo(rt reflect.Type) *typeInfo { // contain things such as private fields, channels, and functions, // which are not usually transmissible in gob streams. // -// Note: Since gobs can be stored permanently, It is good design +// Note: Since gobs can be stored permanently, it is good design // to guarantee the encoding used by a GobEncoder is stable as the // software evolves. For instance, it might make sense for GobEncode // to include a version number in the encoding. diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index 530e852..539d952 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -37,6 +37,7 @@ import ( // To unmarshal JSON into a struct, Unmarshal matches incoming object // keys to the keys used by Marshal (either the struct field name or its tag), // preferring an exact match but also accepting a case-insensitive match. +// Unmarshal will only set exported fields of the struct. // // To unmarshal JSON into an interface value, // Unmarshal stores one of these in the interface value: @@ -48,16 +49,26 @@ import ( // map[string]interface{}, for JSON objects // nil for JSON null // -// To unmarshal a JSON array into a slice, Unmarshal resets the slice to nil -// and then appends each element to the slice. +// To unmarshal a JSON array into a slice, Unmarshal resets the slice length +// to zero and then appends each element to the slice. +// As a special case, to unmarshal an empty JSON array into a slice, +// Unmarshal replaces the slice with a new empty slice. // -// To unmarshal a JSON object into a map, Unmarshal replaces the map -// with an empty map and then adds key-value pairs from the object to -// the map. +// To unmarshal a JSON array into a Go array, Unmarshal decodes +// JSON array elements into corresponding Go array elements. +// If the Go array is smaller than the JSON array, +// the additional JSON array elements are discarded. +// If the JSON array is smaller than the Go array, +// the additional Go array elements are set to zero values. +// +// To unmarshal a JSON object into a string-keyed map, Unmarshal first +// establishes a map to use, If the map is nil, Unmarshal allocates a new map. +// Otherwise Unmarshal reuses the existing map, keeping existing entries. +// Unmarshal then stores key-value pairs from the JSON object into the map. // // If a JSON value is not appropriate for a given target type, // or if a JSON number overflows the target type, Unmarshal -// skips that field and completes the unmarshalling as best it can. +// skips that field and completes the unmarshaling as best it can. // If no more serious errors are encountered, Unmarshal returns // an UnmarshalTypeError describing the earliest such error. // @@ -174,6 +185,66 @@ func (n Number) Int64() (int64, error) { return strconv.ParseInt(string(n), 10, 64) } +// isValidNumber reports whether s is a valid JSON number literal. +func isValidNumber(s string) bool { + // This function implements the JSON numbers grammar. + // See https://tools.ietf.org/html/rfc7159#section-6 + // and http://json.org/number.gif + + if s == "" { + return false + } + + // Optional - + if s[0] == '-' { + s = s[1:] + if s == "" { + return false + } + } + + // Digits + switch { + default: + return false + + case s[0] == '0': + s = s[1:] + + case '1' <= s[0] && s[0] <= '9': + s = s[1:] + for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { + s = s[1:] + } + } + + // . followed by 1 or more digits. + if len(s) >= 2 && s[0] == '.' && '0' <= s[1] && s[1] <= '9' { + s = s[2:] + for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { + s = s[1:] + } + } + + // e or E followed by an optional - or + and + // 1 or more digits. + if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') { + s = s[1:] + if s[0] == '+' || s[0] == '-' { + s = s[1:] + if s == "" { + return false + } + } + for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { + s = s[1:] + } + } + + // Make sure we are at the end. + return s == "" +} + // decodeState represents the state while decoding a JSON value. type decodeState struct { data []byte @@ -241,7 +312,7 @@ func (d *decodeState) scanWhile(op int) int { newOp = d.scan.eof() d.off = len(d.data) + 1 // mark processed EOF with len+1 } else { - c := int(d.data[d.off]) + c := d.data[d.off] d.off++ newOp = d.scan.step(&d.scan, c) } @@ -757,7 +828,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool d.saveError(err) break } - v.Set(reflect.ValueOf(b[0:n])) + v.SetBytes(b[:n]) case reflect.String: v.SetString(string(s)) case reflect.Interface: @@ -781,6 +852,9 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool default: if v.Kind() == reflect.String && v.Type() == numberType { v.SetString(s) + if !isValidNumber(s) { + d.error(fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item)) + } break } if fromQuoted { diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index 51b15ef..9546ae4 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -728,7 +728,7 @@ func TestErrorMessageFromMisusedString(t *testing.T) { } func noSpace(c rune) rune { - if isSpace(c) { + if isSpace(byte(c)) { //only used for ascii return -1 } return c @@ -1218,12 +1218,12 @@ func TestStringKind(t *testing.T) { data, err := Marshal(m1) if err != nil { - t.Errorf("Unexpected error marshalling: %v", err) + t.Errorf("Unexpected error marshaling: %v", err) } err = Unmarshal(data, &m2) if err != nil { - t.Errorf("Unexpected error unmarshalling: %v", err) + t.Errorf("Unexpected error unmarshaling: %v", err) } if !reflect.DeepEqual(m1, m2) { @@ -1253,6 +1253,27 @@ func TestByteKind(t *testing.T) { } } +// The fix for issue 8962 introduced a regression. +// Issue 12921. +func TestSliceOfCustomByte(t *testing.T) { + type Uint8 uint8 + + a := []Uint8("hello") + + data, err := Marshal(a) + if err != nil { + t.Fatal(err) + } + var b []Uint8 + err = Unmarshal(data, &b) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(a, b) { + t.Fatal("expected %v == %v", a, b) + } +} + var decodeTypeErrorTests = []struct { dest interface{} src string diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index e829a93..69ac7e0 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -14,6 +14,7 @@ import ( "bytes" "encoding" "encoding/base64" + "fmt" "math" "reflect" "runtime" @@ -30,7 +31,10 @@ import ( // Marshal traverses the value v recursively. // If an encountered value implements the Marshaler interface // and is not a nil pointer, Marshal calls its MarshalJSON method -// to produce JSON. The nil pointer exception is not strictly necessary +// to produce JSON. If no MarshalJSON method is present but the +// value implements encoding.TextMarshaler instead, Marshal calls +// its MarshalText method. +// The nil pointer exception is not strictly necessary // but mimics a similar, necessary exception in the behavior of // UnmarshalJSON. // @@ -445,12 +449,10 @@ func textMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { } m := v.Interface().(encoding.TextMarshaler) b, err := m.MarshalText() - if err == nil { - _, err = e.stringBytes(b) - } if err != nil { e.error(&MarshalerError{v.Type(), err}) } + e.stringBytes(b) } func addrTextMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { @@ -461,12 +463,10 @@ func addrTextMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { } m := va.Interface().(encoding.TextMarshaler) b, err := m.MarshalText() - if err == nil { - _, err = e.stringBytes(b) - } if err != nil { e.error(&MarshalerError{v.Type(), err}) } + e.stringBytes(b) } func boolEncoder(e *encodeState, v reflect.Value, quoted bool) { @@ -530,9 +530,14 @@ var ( func stringEncoder(e *encodeState, v reflect.Value, quoted bool) { if v.Type() == numberType { numStr := v.String() + // In Go1.5 the empty string encodes to "0", while this is not a valid number literal + // we keep compatibility so check validity after this. if numStr == "" { numStr = "0" // Number's zero-val } + if !isValidNumber(numStr) { + e.error(fmt.Errorf("json: invalid number literal %q", numStr)) + } e.WriteString(numStr) return } @@ -780,7 +785,7 @@ func (sv stringValues) Less(i, j int) bool { return sv.get(i) < sv.get(j) } func (sv stringValues) get(i int) string { return sv[i].String() } // NOTE: keep in sync with stringBytes below. -func (e *encodeState) string(s string) (int, error) { +func (e *encodeState) string(s string) int { len0 := e.Len() e.WriteByte('"') start := 0 @@ -852,11 +857,11 @@ func (e *encodeState) string(s string) (int, error) { e.WriteString(s[start:]) } e.WriteByte('"') - return e.Len() - len0, nil + return e.Len() - len0 } // NOTE: keep in sync with string above. -func (e *encodeState) stringBytes(s []byte) (int, error) { +func (e *encodeState) stringBytes(s []byte) int { len0 := e.Len() e.WriteByte('"') start := 0 @@ -928,7 +933,7 @@ func (e *encodeState) stringBytes(s []byte) (int, error) { e.Write(s[start:]) } e.WriteByte('"') - return e.Len() - len0, nil + return e.Len() - len0 } // A field represents a single field found in a struct. diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index 7abfa85..c00491e 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -381,16 +381,10 @@ func TestStringBytes(t *testing.T) { r = append(r, i) } s := string(r) + "\xff\xff\xffhello" // some invalid UTF-8 too - _, err := es.string(s) - if err != nil { - t.Fatal(err) - } + es.string(s) esBytes := &encodeState{} - _, err = esBytes.stringBytes([]byte(s)) - if err != nil { - t.Fatal(err) - } + esBytes.stringBytes([]byte(s)) enc := es.Buffer.String() encBytes := esBytes.Buffer.String() @@ -443,6 +437,18 @@ func TestIssue6458(t *testing.T) { } } +func TestIssue10281(t *testing.T) { + type Foo struct { + N Number + } + x := Foo{Number(`invalid`)} + + b, err := Marshal(&x) + if err == nil { + t.Errorf("Marshal(&x) = %#q; want error", b) + } +} + func TestHTMLEscape(t *testing.T) { var b, want bytes.Buffer m := `{"M":"<html>foo &` + "\xe2\x80\xa8 \xe2\x80\xa9" + `</html>"}` diff --git a/libgo/go/encoding/json/indent.go b/libgo/go/encoding/json/indent.go index e1bacaf..7cd9f4d 100644 --- a/libgo/go/encoding/json/indent.go +++ b/libgo/go/encoding/json/indent.go @@ -36,7 +36,7 @@ func compact(dst *bytes.Buffer, src []byte, escape bool) error { dst.WriteByte(hex[src[i+2]&0xF]) start = i + 3 } - v := scan.step(&scan, int(c)) + v := scan.step(&scan, c) if v >= scanSkipSpace { if v == scanError { break @@ -70,8 +70,12 @@ func newline(dst *bytes.Buffer, prefix, indent string, depth int) { // indented line beginning with prefix followed by one or more // copies of indent according to the indentation nesting. // The data appended to dst does not begin with the prefix nor -// any indentation, and has no trailing newline, to make it -// easier to embed inside other formatted JSON data. +// any indentation, to make it easier to embed inside other formatted JSON data. +// Although leading space characters (space, tab, carriage return, newline) +// at the beginning of src are dropped, trailing space characters +// at the end of src are preserved and copied to dst. +// For example, if src has no trailing spaces, neither will dst; +// if src ends in a trailing newline, so will dst. func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { origLen := dst.Len() var scan scanner @@ -80,7 +84,7 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { depth := 0 for _, c := range src { scan.bytes++ - v := scan.step(&scan, int(c)) + v := scan.step(&scan, c) if v == scanSkipSpace { continue } diff --git a/libgo/go/encoding/json/number_test.go b/libgo/go/encoding/json/number_test.go new file mode 100644 index 0000000..4e63cf9 --- /dev/null +++ b/libgo/go/encoding/json/number_test.go @@ -0,0 +1,133 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "regexp" + "testing" +) + +func TestNumberIsValid(t *testing.T) { + // From: http://stackoverflow.com/a/13340826 + var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) + + validTests := []string{ + "0", + "-0", + "1", + "-1", + "0.1", + "-0.1", + "1234", + "-1234", + "12.34", + "-12.34", + "12E0", + "12E1", + "12e34", + "12E-0", + "12e+1", + "12e-34", + "-12E0", + "-12E1", + "-12e34", + "-12E-0", + "-12e+1", + "-12e-34", + "1.2E0", + "1.2E1", + "1.2e34", + "1.2E-0", + "1.2e+1", + "1.2e-34", + "-1.2E0", + "-1.2E1", + "-1.2e34", + "-1.2E-0", + "-1.2e+1", + "-1.2e-34", + "0E0", + "0E1", + "0e34", + "0E-0", + "0e+1", + "0e-34", + "-0E0", + "-0E1", + "-0e34", + "-0E-0", + "-0e+1", + "-0e-34", + } + + for _, test := range validTests { + if !isValidNumber(test) { + t.Errorf("%s should be valid", test) + } + + var f float64 + if err := Unmarshal([]byte(test), &f); err != nil { + t.Errorf("%s should be valid but Unmarshal failed: %v", test, err) + } + + if !jsonNumberRegexp.MatchString(test) { + t.Errorf("%s should be valid but regexp does not match", test) + } + } + + invalidTests := []string{ + "", + "invalid", + "1.0.1", + "1..1", + "-1-2", + "012a42", + "01.2", + "012", + "12E12.12", + "1e2e3", + "1e+-2", + "1e--23", + "1e", + "e1", + "1e+", + "1ea", + "1a", + "1.a", + "1.", + "01", + "1.e1", + } + + for _, test := range invalidTests { + if isValidNumber(test) { + t.Errorf("%s should be invalid", test) + } + + var f float64 + if err := Unmarshal([]byte(test), &f); err == nil { + t.Errorf("%s should be invalid but unmarshal wrote %v", test, f) + } + + if jsonNumberRegexp.MatchString(test) { + t.Errorf("%s should be invalid but matches regexp", test) + } + } +} + +func BenchmarkNumberIsValid(b *testing.B) { + s := "-61657.61667E+61673" + for i := 0; i < b.N; i++ { + isValidNumber(s) + } +} + +func BenchmarkNumberIsValidRegexp(b *testing.B) { + var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) + s := "-61657.61667E+61673" + for i := 0; i < b.N; i++ { + jsonNumberRegexp.MatchString(s) + } +} diff --git a/libgo/go/encoding/json/scanner.go b/libgo/go/encoding/json/scanner.go index 38d0b08..ee6622e 100644 --- a/libgo/go/encoding/json/scanner.go +++ b/libgo/go/encoding/json/scanner.go @@ -21,7 +21,7 @@ func checkValid(data []byte, scan *scanner) error { scan.reset() for _, c := range data { scan.bytes++ - if scan.step(scan, int(c)) == scanError { + if scan.step(scan, c) == scanError { return scan.err } } @@ -37,7 +37,7 @@ func checkValid(data []byte, scan *scanner) error { func nextValue(data []byte, scan *scanner) (value, rest []byte, err error) { scan.reset() for i, c := range data { - v := scan.step(scan, int(c)) + v := scan.step(scan, c) if v >= scanEndObject { switch v { // probe the scanner with a space to determine whether we will @@ -50,7 +50,7 @@ func nextValue(data []byte, scan *scanner) (value, rest []byte, err error) { case scanError: return nil, nil, scan.err case scanEnd: - return data[0:i], data[i:], nil + return data[:i], data[i:], nil } } } @@ -85,7 +85,7 @@ type scanner struct { // Also tried using an integer constant and a single func // with a switch, but using the func directly was 10% faster // on a 64-bit Mac Mini, and it's nicer to read. - step func(*scanner, int) int + step func(*scanner, byte) int // Reached end of top-level value. endTop bool @@ -99,7 +99,7 @@ type scanner struct { // 1-byte redo (see undo method) redo bool redoCode int - redoState func(*scanner, int) int + redoState func(*scanner, byte) int // total bytes consumed, updated by decoder.Decode bytes int64 @@ -188,13 +188,13 @@ func (s *scanner) popParseState() { } } -func isSpace(c rune) bool { +func isSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n' } // stateBeginValueOrEmpty is the state after reading `[`. -func stateBeginValueOrEmpty(s *scanner, c int) int { - if c <= ' ' && isSpace(rune(c)) { +func stateBeginValueOrEmpty(s *scanner, c byte) int { + if c <= ' ' && isSpace(c) { return scanSkipSpace } if c == ']' { @@ -204,8 +204,8 @@ func stateBeginValueOrEmpty(s *scanner, c int) int { } // stateBeginValue is the state at the beginning of the input. -func stateBeginValue(s *scanner, c int) int { - if c <= ' ' && isSpace(rune(c)) { +func stateBeginValue(s *scanner, c byte) int { + if c <= ' ' && isSpace(c) { return scanSkipSpace } switch c { @@ -244,8 +244,8 @@ func stateBeginValue(s *scanner, c int) int { } // stateBeginStringOrEmpty is the state after reading `{`. -func stateBeginStringOrEmpty(s *scanner, c int) int { - if c <= ' ' && isSpace(rune(c)) { +func stateBeginStringOrEmpty(s *scanner, c byte) int { + if c <= ' ' && isSpace(c) { return scanSkipSpace } if c == '}' { @@ -257,8 +257,8 @@ func stateBeginStringOrEmpty(s *scanner, c int) int { } // stateBeginString is the state after reading `{"key": value,`. -func stateBeginString(s *scanner, c int) int { - if c <= ' ' && isSpace(rune(c)) { +func stateBeginString(s *scanner, c byte) int { + if c <= ' ' && isSpace(c) { return scanSkipSpace } if c == '"' { @@ -270,7 +270,7 @@ func stateBeginString(s *scanner, c int) int { // stateEndValue is the state after completing a value, // such as after reading `{}` or `true` or `["x"`. -func stateEndValue(s *scanner, c int) int { +func stateEndValue(s *scanner, c byte) int { n := len(s.parseState) if n == 0 { // Completed top-level before the current byte. @@ -278,7 +278,7 @@ func stateEndValue(s *scanner, c int) int { s.endTop = true return stateEndTop(s, c) } - if c <= ' ' && isSpace(rune(c)) { + if c <= ' ' && isSpace(c) { s.step = stateEndValue return scanSkipSpace } @@ -319,7 +319,7 @@ func stateEndValue(s *scanner, c int) int { // stateEndTop is the state after finishing the top-level value, // such as after reading `{}` or `[1,2,3]`. // Only space characters should be seen now. -func stateEndTop(s *scanner, c int) int { +func stateEndTop(s *scanner, c byte) int { if c != ' ' && c != '\t' && c != '\r' && c != '\n' { // Complain about non-space byte on next call. s.error(c, "after top-level value") @@ -328,7 +328,7 @@ func stateEndTop(s *scanner, c int) int { } // stateInString is the state after reading `"`. -func stateInString(s *scanner, c int) int { +func stateInString(s *scanner, c byte) int { if c == '"' { s.step = stateEndValue return scanContinue @@ -344,13 +344,12 @@ func stateInString(s *scanner, c int) int { } // stateInStringEsc is the state after reading `"\` during a quoted string. -func stateInStringEsc(s *scanner, c int) int { +func stateInStringEsc(s *scanner, c byte) int { switch c { case 'b', 'f', 'n', 'r', 't', '\\', '/', '"': s.step = stateInString return scanContinue - } - if c == 'u' { + case 'u': s.step = stateInStringEscU return scanContinue } @@ -358,7 +357,7 @@ func stateInStringEsc(s *scanner, c int) int { } // stateInStringEscU is the state after reading `"\u` during a quoted string. -func stateInStringEscU(s *scanner, c int) int { +func stateInStringEscU(s *scanner, c byte) int { if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { s.step = stateInStringEscU1 return scanContinue @@ -368,7 +367,7 @@ func stateInStringEscU(s *scanner, c int) int { } // stateInStringEscU1 is the state after reading `"\u1` during a quoted string. -func stateInStringEscU1(s *scanner, c int) int { +func stateInStringEscU1(s *scanner, c byte) int { if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { s.step = stateInStringEscU12 return scanContinue @@ -378,7 +377,7 @@ func stateInStringEscU1(s *scanner, c int) int { } // stateInStringEscU12 is the state after reading `"\u12` during a quoted string. -func stateInStringEscU12(s *scanner, c int) int { +func stateInStringEscU12(s *scanner, c byte) int { if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { s.step = stateInStringEscU123 return scanContinue @@ -388,7 +387,7 @@ func stateInStringEscU12(s *scanner, c int) int { } // stateInStringEscU123 is the state after reading `"\u123` during a quoted string. -func stateInStringEscU123(s *scanner, c int) int { +func stateInStringEscU123(s *scanner, c byte) int { if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { s.step = stateInString return scanContinue @@ -398,7 +397,7 @@ func stateInStringEscU123(s *scanner, c int) int { } // stateNeg is the state after reading `-` during a number. -func stateNeg(s *scanner, c int) int { +func stateNeg(s *scanner, c byte) int { if c == '0' { s.step = state0 return scanContinue @@ -412,7 +411,7 @@ func stateNeg(s *scanner, c int) int { // state1 is the state after reading a non-zero integer during a number, // such as after reading `1` or `100` but not `0`. -func state1(s *scanner, c int) int { +func state1(s *scanner, c byte) int { if '0' <= c && c <= '9' { s.step = state1 return scanContinue @@ -421,7 +420,7 @@ func state1(s *scanner, c int) int { } // state0 is the state after reading `0` during a number. -func state0(s *scanner, c int) int { +func state0(s *scanner, c byte) int { if c == '.' { s.step = stateDot return scanContinue @@ -435,7 +434,7 @@ func state0(s *scanner, c int) int { // stateDot is the state after reading the integer and decimal point in a number, // such as after reading `1.`. -func stateDot(s *scanner, c int) int { +func stateDot(s *scanner, c byte) int { if '0' <= c && c <= '9' { s.step = stateDot0 return scanContinue @@ -445,9 +444,8 @@ func stateDot(s *scanner, c int) int { // stateDot0 is the state after reading the integer, decimal point, and subsequent // digits of a number, such as after reading `3.14`. -func stateDot0(s *scanner, c int) int { +func stateDot0(s *scanner, c byte) int { if '0' <= c && c <= '9' { - s.step = stateDot0 return scanContinue } if c == 'e' || c == 'E' { @@ -459,12 +457,8 @@ func stateDot0(s *scanner, c int) int { // stateE is the state after reading the mantissa and e in a number, // such as after reading `314e` or `0.314e`. -func stateE(s *scanner, c int) int { - if c == '+' { - s.step = stateESign - return scanContinue - } - if c == '-' { +func stateE(s *scanner, c byte) int { + if c == '+' || c == '-' { s.step = stateESign return scanContinue } @@ -473,7 +467,7 @@ func stateE(s *scanner, c int) int { // stateESign is the state after reading the mantissa, e, and sign in a number, // such as after reading `314e-` or `0.314e+`. -func stateESign(s *scanner, c int) int { +func stateESign(s *scanner, c byte) int { if '0' <= c && c <= '9' { s.step = stateE0 return scanContinue @@ -484,16 +478,15 @@ func stateESign(s *scanner, c int) int { // stateE0 is the state after reading the mantissa, e, optional sign, // and at least one digit of the exponent in a number, // such as after reading `314e-2` or `0.314e+1` or `3.14e0`. -func stateE0(s *scanner, c int) int { +func stateE0(s *scanner, c byte) int { if '0' <= c && c <= '9' { - s.step = stateE0 return scanContinue } return stateEndValue(s, c) } // stateT is the state after reading `t`. -func stateT(s *scanner, c int) int { +func stateT(s *scanner, c byte) int { if c == 'r' { s.step = stateTr return scanContinue @@ -502,7 +495,7 @@ func stateT(s *scanner, c int) int { } // stateTr is the state after reading `tr`. -func stateTr(s *scanner, c int) int { +func stateTr(s *scanner, c byte) int { if c == 'u' { s.step = stateTru return scanContinue @@ -511,7 +504,7 @@ func stateTr(s *scanner, c int) int { } // stateTru is the state after reading `tru`. -func stateTru(s *scanner, c int) int { +func stateTru(s *scanner, c byte) int { if c == 'e' { s.step = stateEndValue return scanContinue @@ -520,7 +513,7 @@ func stateTru(s *scanner, c int) int { } // stateF is the state after reading `f`. -func stateF(s *scanner, c int) int { +func stateF(s *scanner, c byte) int { if c == 'a' { s.step = stateFa return scanContinue @@ -529,7 +522,7 @@ func stateF(s *scanner, c int) int { } // stateFa is the state after reading `fa`. -func stateFa(s *scanner, c int) int { +func stateFa(s *scanner, c byte) int { if c == 'l' { s.step = stateFal return scanContinue @@ -538,7 +531,7 @@ func stateFa(s *scanner, c int) int { } // stateFal is the state after reading `fal`. -func stateFal(s *scanner, c int) int { +func stateFal(s *scanner, c byte) int { if c == 's' { s.step = stateFals return scanContinue @@ -547,7 +540,7 @@ func stateFal(s *scanner, c int) int { } // stateFals is the state after reading `fals`. -func stateFals(s *scanner, c int) int { +func stateFals(s *scanner, c byte) int { if c == 'e' { s.step = stateEndValue return scanContinue @@ -556,7 +549,7 @@ func stateFals(s *scanner, c int) int { } // stateN is the state after reading `n`. -func stateN(s *scanner, c int) int { +func stateN(s *scanner, c byte) int { if c == 'u' { s.step = stateNu return scanContinue @@ -565,7 +558,7 @@ func stateN(s *scanner, c int) int { } // stateNu is the state after reading `nu`. -func stateNu(s *scanner, c int) int { +func stateNu(s *scanner, c byte) int { if c == 'l' { s.step = stateNul return scanContinue @@ -574,7 +567,7 @@ func stateNu(s *scanner, c int) int { } // stateNul is the state after reading `nul`. -func stateNul(s *scanner, c int) int { +func stateNul(s *scanner, c byte) int { if c == 'l' { s.step = stateEndValue return scanContinue @@ -584,19 +577,19 @@ func stateNul(s *scanner, c int) int { // stateError is the state after reaching a syntax error, // such as after reading `[1}` or `5.1.2`. -func stateError(s *scanner, c int) int { +func stateError(s *scanner, c byte) int { return scanError } // error records an error and switches to the error state. -func (s *scanner) error(c int, context string) int { +func (s *scanner) error(c byte, context string) int { s.step = stateError s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes} return scanError } // quoteChar formats c as a quoted character literal -func quoteChar(c int) string { +func quoteChar(c byte) string { // special cases - different from quoted strings if c == '\'' { return `'\''` @@ -623,7 +616,7 @@ func (s *scanner) undo(scanCode int) { } // stateRedo helps implement the scanner's 1-byte undo. -func stateRedo(s *scanner, c int) int { +func stateRedo(s *scanner, c byte) int { s.redo = false s.step = s.redoState return s.redoCode diff --git a/libgo/go/encoding/json/stream.go b/libgo/go/encoding/json/stream.go index dc53bce..8ddcf4d 100644 --- a/libgo/go/encoding/json/stream.go +++ b/libgo/go/encoding/json/stream.go @@ -90,7 +90,7 @@ Input: // Look in the buffer for a new value. for i, c := range dec.buf[scanp:] { dec.scan.bytes++ - v := dec.scan.step(&dec.scan, int(c)) + v := dec.scan.step(&dec.scan, c) if v == scanEnd { scanp += i break Input @@ -157,7 +157,7 @@ func (dec *Decoder) refill() error { func nonSpace(b []byte) bool { for _, c := range b { - if !isSpace(rune(c)) { + if !isSpace(c) { return true } } @@ -433,7 +433,7 @@ func (dec *Decoder) tokenError(c byte) (Token, error) { case tokenObjectComma: context = " after object key:value pair" } - return nil, &SyntaxError{"invalid character " + quoteChar(int(c)) + " " + context, 0} + return nil, &SyntaxError{"invalid character " + quoteChar(c) + " " + context, 0} } // More reports whether there is another element in the @@ -448,7 +448,7 @@ func (dec *Decoder) peek() (byte, error) { for { for i := dec.scanp; i < len(dec.buf); i++ { c := dec.buf[i] - if isSpace(rune(c)) { + if isSpace(c) { continue } dec.scanp = i diff --git a/libgo/go/encoding/pem/pem_test.go b/libgo/go/encoding/pem/pem_test.go index ab656c6..958dbc1 100644 --- a/libgo/go/encoding/pem/pem_test.go +++ b/libgo/go/encoding/pem/pem_test.go @@ -155,20 +155,28 @@ func TestFuzz(t *testing.T) { } var buf bytes.Buffer - err := Encode(&buf, &block) - decoded, rest := Decode(buf.Bytes()) - - switch { - case err != nil: + if err := Encode(&buf, &block); err != nil { t.Errorf("Encode of %#v resulted in error: %s", &block, err) - case !reflect.DeepEqual(&block, decoded): + return false + } + decoded, rest := Decode(buf.Bytes()) + if block.Headers == nil { + // Encoder supports nil Headers but decoder returns initialized. + block.Headers = make(map[string]string) + } + if block.Bytes == nil { + // Encoder supports nil Bytes but decoder returns initialized. + block.Bytes = make([]byte, 0) + } + if !reflect.DeepEqual(decoded, &block) { t.Errorf("Encode of %#v decoded as %#v", &block, decoded) - case len(rest) != 0: + return false + } + if len(rest) != 0 { t.Errorf("Encode of %#v decoded correctly, but with %x left over", block, rest) - default: - return true + return false } - return false + return true } // Explicitly test the empty block. diff --git a/libgo/go/encoding/xml/marshal.go b/libgo/go/encoding/xml/marshal.go index 86d1422..8ebd693 100644 --- a/libgo/go/encoding/xml/marshal.go +++ b/libgo/go/encoding/xml/marshal.go @@ -48,6 +48,8 @@ const ( // field name in the XML element. // - a field with tag ",chardata" is written as character data, // not as an XML element. +// - a field with tag ",cdata" is written as character data +// wrapped in one or more <![CDATA[ ... ]]> tags, not as an XML element. // - a field with tag ",innerxml" is written verbatim, not subject // to the usual marshalling procedure. // - a field with tag ",comment" is written as an XML comment, not @@ -768,7 +770,11 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { } switch finfo.flags & fMode { - case fCharData: + case fCDATA, fCharData: + emit := EscapeText + if finfo.flags&fMode == fCDATA { + emit = emitCDATA + } if err := s.trim(finfo.parents); err != nil { return err } @@ -777,7 +783,9 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { if err != nil { return err } - Escape(p, data) + if err := emit(p, data); err != nil { + return err + } continue } if vf.CanAddr() { @@ -787,27 +795,37 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { if err != nil { return err } - Escape(p, data) + if err := emit(p, data); err != nil { + return err + } continue } } var scratch [64]byte switch vf.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - Escape(p, strconv.AppendInt(scratch[:0], vf.Int(), 10)) + if err := emit(p, strconv.AppendInt(scratch[:0], vf.Int(), 10)); err != nil { + return err + } case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - Escape(p, strconv.AppendUint(scratch[:0], vf.Uint(), 10)) + if err := emit(p, strconv.AppendUint(scratch[:0], vf.Uint(), 10)); err != nil { + return err + } case reflect.Float32, reflect.Float64: - Escape(p, strconv.AppendFloat(scratch[:0], vf.Float(), 'g', -1, vf.Type().Bits())) + if err := emit(p, strconv.AppendFloat(scratch[:0], vf.Float(), 'g', -1, vf.Type().Bits())); err != nil { + return err + } case reflect.Bool: - Escape(p, strconv.AppendBool(scratch[:0], vf.Bool())) + if err := emit(p, strconv.AppendBool(scratch[:0], vf.Bool())); err != nil { + return err + } case reflect.String: - if err := EscapeText(p, []byte(vf.String())); err != nil { + if err := emit(p, []byte(vf.String())); err != nil { return err } case reflect.Slice: if elem, ok := vf.Interface().([]byte); ok { - if err := EscapeText(p, elem); err != nil { + if err := emit(p, elem); err != nil { return err } } diff --git a/libgo/go/encoding/xml/marshal_test.go b/libgo/go/encoding/xml/marshal_test.go index ef6c20e..fe8b16f 100644 --- a/libgo/go/encoding/xml/marshal_test.go +++ b/libgo/go/encoding/xml/marshal_test.go @@ -356,6 +356,15 @@ type NestedAndComment struct { Comment string `xml:",comment"` } +type CDataTest struct { + Chardata string `xml:",cdata"` +} + +type NestedAndCData struct { + AB []string `xml:"A>B"` + CDATA string `xml:",cdata"` +} + func ifaceptr(x interface{}) interface{} { return &x } @@ -978,6 +987,42 @@ var marshalTests = []struct { MyInt: 42, }, }, + // Test outputting CDATA-wrapped text. + { + ExpectXML: `<CDataTest></CDataTest>`, + Value: &CDataTest{}, + }, + { + ExpectXML: `<CDataTest><![CDATA[http://example.com/tests/1?foo=1&bar=baz]]></CDataTest>`, + Value: &CDataTest{ + Chardata: "http://example.com/tests/1?foo=1&bar=baz", + }, + }, + { + ExpectXML: `<CDataTest><![CDATA[Literal <![CDATA[Nested]]]]><![CDATA[>!]]></CDataTest>`, + Value: &CDataTest{ + Chardata: "Literal <![CDATA[Nested]]>!", + }, + }, + { + ExpectXML: `<CDataTest><![CDATA[<![CDATA[Nested]]]]><![CDATA[> Literal!]]></CDataTest>`, + Value: &CDataTest{ + Chardata: "<![CDATA[Nested]]> Literal!", + }, + }, + { + ExpectXML: `<CDataTest><![CDATA[<![CDATA[Nested]]]]><![CDATA[> Literal! <![CDATA[Nested]]]]><![CDATA[> Literal!]]></CDataTest>`, + Value: &CDataTest{ + Chardata: "<![CDATA[Nested]]> Literal! <![CDATA[Nested]]> Literal!", + }, + }, + { + ExpectXML: `<CDataTest><![CDATA[<![CDATA[<![CDATA[Nested]]]]><![CDATA[>]]]]><![CDATA[>]]></CDataTest>`, + Value: &CDataTest{ + Chardata: "<![CDATA[<![CDATA[Nested]]>]]>", + }, + }, + // Test omitempty with parent chain; see golang.org/issue/4168. { ExpectXML: `<Strings><A></A></Strings>`, @@ -1016,6 +1061,10 @@ var marshalTests = []struct { ExpectXML: `<NestedAndComment><A><B></B><B></B></A><!--test--></NestedAndComment>`, Value: &NestedAndComment{AB: make([]string, 2), Comment: "test"}, }, + { + ExpectXML: `<NestedAndCData><A><B></B><B></B></A><![CDATA[test]]></NestedAndCData>`, + Value: &NestedAndCData{AB: make([]string, 2), CDATA: "test"}, + }, } func TestMarshal(t *testing.T) { @@ -1705,7 +1754,7 @@ func TestRace9796(t *testing.T) { for i := 0; i < 2; i++ { wg.Add(1) go func() { - Marshal(B{[]A{A{}}}) + Marshal(B{[]A{{}}}) wg.Done() }() } diff --git a/libgo/go/encoding/xml/read.go b/libgo/go/encoding/xml/read.go index 75b9f2b..77b4c7b 100644 --- a/libgo/go/encoding/xml/read.go +++ b/libgo/go/encoding/xml/read.go @@ -431,7 +431,7 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { } } - case fCharData: + case fCDATA, fCharData: if !saveData.IsValid() { saveData = finfo.value(sv) } diff --git a/libgo/go/encoding/xml/read_test.go b/libgo/go/encoding/xml/read_test.go index 7d004dc..7a98092 100644 --- a/libgo/go/encoding/xml/read_test.go +++ b/libgo/go/encoding/xml/read_test.go @@ -712,3 +712,24 @@ func TestUnmarshalIntoInterface(t *testing.T) { t.Errorf("failed to unmarshal into interface, have %q want %q", have, want) } } + +type X struct { + D string `xml:",comment"` +} + +// Issue 11112. Unmarshal must reject invalid comments. +func TestMalformedComment(t *testing.T) { + testData := []string{ + "<X><!-- a---></X>", + "<X><!-- -- --></X>", + "<X><!-- a--b --></X>", + "<X><!------></X>", + } + for i, test := range testData { + data := []byte(test) + v := new(X) + if err := Unmarshal(data, v); err == nil { + t.Errorf("%d: unmarshal should reject invalid comments", i) + } + } +} diff --git a/libgo/go/encoding/xml/typeinfo.go b/libgo/go/encoding/xml/typeinfo.go index 6766b88..6483c8d 100644 --- a/libgo/go/encoding/xml/typeinfo.go +++ b/libgo/go/encoding/xml/typeinfo.go @@ -31,6 +31,7 @@ type fieldFlags int const ( fElement fieldFlags = 1 << iota fAttr + fCDATA fCharData fInnerXml fComment @@ -38,7 +39,7 @@ const ( fOmitEmpty - fMode = fElement | fAttr | fCharData | fInnerXml | fComment | fAny + fMode = fElement | fAttr | fCDATA | fCharData | fInnerXml | fComment | fAny ) var tinfoMap = make(map[reflect.Type]*typeInfo) @@ -130,6 +131,8 @@ func structFieldInfo(typ reflect.Type, f *reflect.StructField) (*fieldInfo, erro switch flag { case "attr": finfo.flags |= fAttr + case "cdata": + finfo.flags |= fCDATA case "chardata": finfo.flags |= fCharData case "innerxml": @@ -148,7 +151,7 @@ func structFieldInfo(typ reflect.Type, f *reflect.StructField) (*fieldInfo, erro switch mode := finfo.flags & fMode; mode { case 0: finfo.flags |= fElement - case fAttr, fCharData, fInnerXml, fComment, fAny: + case fAttr, fCDATA, fCharData, fInnerXml, fComment, fAny: if f.Name == "XMLName" || tag != "" && mode != fAttr { valid = false } diff --git a/libgo/go/encoding/xml/xml.go b/libgo/go/encoding/xml/xml.go index 0a21c93..45f4157 100644 --- a/libgo/go/encoding/xml/xml.go +++ b/libgo/go/encoding/xml/xml.go @@ -227,7 +227,8 @@ func NewDecoder(r io.Reader) *Decoder { // // Token guarantees that the StartElement and EndElement // tokens it returns are properly nested and matched: -// if Token encounters an unexpected end element, +// if Token encounters an unexpected end element +// or EOF before all expected end elements, // it will return an error. // // Token implements XML name spaces as described by @@ -245,6 +246,9 @@ func (d *Decoder) Token() (t Token, err error) { t = d.nextToken d.nextToken = nil } else if t, err = d.rawToken(); err != nil { + if err == io.EOF && d.stk != nil && d.stk.kind != stkEOF { + err = d.syntaxError("unexpected EOF") + } return } @@ -580,7 +584,7 @@ func (d *Decoder) rawToken() (Token, error) { return nil, d.err } enc := procInst("encoding", content) - if enc != "" && enc != "utf-8" && enc != "UTF-8" { + if enc != "" && enc != "utf-8" && enc != "UTF-8" && !strings.EqualFold(enc, "utf-8") { if d.CharsetReader == nil { d.err = fmt.Errorf("xml: encoding %q declared but Decoder.CharsetReader is nil", enc) return nil, d.err @@ -621,7 +625,12 @@ func (d *Decoder) rawToken() (Token, error) { return nil, d.err } d.buf.WriteByte(b) - if b0 == '-' && b1 == '-' && b == '>' { + if b0 == '-' && b1 == '-' { + if b != '>' { + d.err = d.syntaxError( + `invalid sequence "--" not allowed in comments`) + return nil, d.err + } break } b0, b1 = b1, b @@ -1940,6 +1949,46 @@ func Escape(w io.Writer, s []byte) { EscapeText(w, s) } +var ( + cdataStart = []byte("<![CDATA[") + cdataEnd = []byte("]]>") + cdataEscape = []byte("]]]]><![CDATA[>") +) + +// emitCDATA writes to w the CDATA-wrapped plain text data s. +// It escapes CDATA directives nested in s. +func emitCDATA(w io.Writer, s []byte) error { + if len(s) == 0 { + return nil + } + if _, err := w.Write(cdataStart); err != nil { + return err + } + for { + i := bytes.Index(s, cdataEnd) + if i >= 0 && i+len(cdataEnd) <= len(s) { + // Found a nested CDATA directive end. + if _, err := w.Write(s[:i]); err != nil { + return err + } + if _, err := w.Write(cdataEscape); err != nil { + return err + } + i += len(cdataEnd) + } else { + if _, err := w.Write(s); err != nil { + return err + } + break + } + s = s[i:] + } + if _, err := w.Write(cdataEnd); err != nil { + return err + } + return nil +} + // procInst parses the `param="..."` or `param='...'` // value out of the provided string, returning "" if not found. func procInst(param, s string) string { diff --git a/libgo/go/encoding/xml/xml_test.go b/libgo/go/encoding/xml/xml_test.go index 312a7c9..5d5e4bf 100644 --- a/libgo/go/encoding/xml/xml_test.go +++ b/libgo/go/encoding/xml/xml_test.go @@ -750,3 +750,56 @@ func TestIssue5880(t *testing.T) { t.Errorf("Marshal generated invalid UTF-8: %x", data) } } + +func TestIssue11405(t *testing.T) { + testCases := []string{ + "<root>", + "<root><foo>", + "<root><foo></foo>", + } + for _, tc := range testCases { + d := NewDecoder(strings.NewReader(tc)) + var err error + for { + _, err = d.Token() + if err != nil { + break + } + } + if _, ok := err.(*SyntaxError); !ok { + t.Errorf("%s: Token: Got error %v, want SyntaxError", tc, err) + } + } +} + +func TestIssue12417(t *testing.T) { + testCases := []struct { + s string + ok bool + }{ + {`<?xml encoding="UtF-8" version="1.0"?><root/>`, true}, + {`<?xml encoding="UTF-8" version="1.0"?><root/>`, true}, + {`<?xml encoding="utf-8" version="1.0"?><root/>`, true}, + {`<?xml encoding="uuu-9" version="1.0"?><root/>`, false}, + } + for _, tc := range testCases { + d := NewDecoder(strings.NewReader(tc.s)) + var err error + for { + _, err = d.Token() + if err != nil { + if err == io.EOF { + err = nil + } + break + } + } + if err != nil && tc.ok { + t.Errorf("%q: Encoding charset: expected no error, got %s", tc.s, err) + continue + } + if err == nil && !tc.ok { + t.Errorf("%q: Encoding charset: expected error, got nil", tc.s) + } + } +} |