diff options
author | Ian Lance Taylor <iant@google.com> | 2015-01-15 00:27:56 +0000 |
---|---|---|
committer | Ian Lance Taylor <ian@gcc.gnu.org> | 2015-01-15 00:27:56 +0000 |
commit | f8d9fa9e80b57f89e7877ce6cad8a3464879009b (patch) | |
tree | 58a1724fee16d2b03c65678c4dd9b50bb97137a9 /libgo/go/encoding | |
parent | 6bd3f109d8d8fa58eeccd6b3504721b4f20c00c2 (diff) | |
download | gcc-f8d9fa9e80b57f89e7877ce6cad8a3464879009b.zip gcc-f8d9fa9e80b57f89e7877ce6cad8a3464879009b.tar.gz gcc-f8d9fa9e80b57f89e7877ce6cad8a3464879009b.tar.bz2 |
libgo, compiler: Upgrade libgo to Go 1.4, except for runtime.
This upgrades all of libgo other than the runtime package to
the Go 1.4 release. In Go 1.4 much of the runtime was
rewritten into Go. Merging that code will take more time and
will not change the API, so I'm putting it off for now.
There are a few runtime changes anyhow, to accomodate other
packages that rely on minor modifications to the runtime
support.
The compiler changes slightly to add a one-bit flag to each
type descriptor kind that is stored directly in an interface,
which for gccgo is currently only pointer types. Another
one-bit flag (gcprog) is reserved because it is used by the gc
compiler, but gccgo does not currently use it.
There is another error check in the compiler since I ran
across it during testing.
gotools/:
* Makefile.am (go_cmd_go_files): Sort entries. Add generate.go.
* Makefile.in: Rebuild.
From-SVN: r219627
Diffstat (limited to 'libgo/go/encoding')
34 files changed, 2863 insertions, 1080 deletions
diff --git a/libgo/go/encoding/ascii85/ascii85.go b/libgo/go/encoding/ascii85/ascii85.go index 60da304..4d71938 100644 --- a/libgo/go/encoding/ascii85/ascii85.go +++ b/libgo/go/encoding/ascii85/ascii85.go @@ -249,7 +249,6 @@ type decoder struct { err error readErr error r io.Reader - end bool // saw end of message buf [1024]byte // leftover input nbuf int out []byte // leftover decoded output diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go index ec7f91c..8b3d1b3 100644 --- a/libgo/go/encoding/asn1/asn1.go +++ b/libgo/go/encoding/asn1/asn1.go @@ -640,15 +640,19 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam // when it sees a string, so if we see a different string type on the // wire, we change the universal type to match. if universalTag == tagPrintableString { - switch t.tag { - case tagIA5String, tagGeneralString, tagT61String, tagUTF8String: - universalTag = t.tag + if t.class == classUniversal { + switch t.tag { + case tagIA5String, tagGeneralString, tagT61String, tagUTF8String: + universalTag = t.tag + } + } else if params.stringType != 0 { + universalTag = params.stringType } } // Special case for time: UTCTime and GeneralizedTime both map to the // Go type time.Time. - if universalTag == tagUTCTime && t.tag == tagGeneralizedTime { + if universalTag == tagUTCTime && t.tag == tagGeneralizedTime && t.class == classUniversal { universalTag = tagGeneralizedTime } @@ -822,8 +826,19 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam return } +// canHaveDefaultValue reports whether k is a Kind that we will set a default +// value for. (A signed integer, essentially.) +func canHaveDefaultValue(k reflect.Kind) bool { + switch k { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return true + } + + return false +} + // setDefaultValue is used to install a default value, from a tag string, into -// a Value. It is successful is the field was optional, even if a default value +// a Value. It is successful if the field was optional, even if a default value // wasn't provided or it failed to install it into the Value. func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { if !params.optional { @@ -833,9 +848,8 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { if params.defaultValue == nil { return } - switch val := v; val.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - val.SetInt(*params.defaultValue) + if canHaveDefaultValue(v.Kind()) { + v.SetInt(*params.defaultValue) } return } diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go index b553f78..4e864d0 100644 --- a/libgo/go/encoding/asn1/asn1_test.go +++ b/libgo/go/encoding/asn1/asn1_test.go @@ -392,6 +392,10 @@ type TestContextSpecificTags2 struct { B int } +type TestContextSpecificTags3 struct { + S string `asn1:"tag:1,utf8"` +} + type TestElementsAfterString struct { S string A, B int @@ -420,6 +424,7 @@ var unmarshalTestData = []struct { {[]byte{0x04, 0x04, 1, 2, 3, 4}, &RawValue{0, 4, false, []byte{1, 2, 3, 4}, []byte{4, 4, 1, 2, 3, 4}}}, {[]byte{0x30, 0x03, 0x81, 0x01, 0x01}, &TestContextSpecificTags{1}}, {[]byte{0x30, 0x08, 0xa1, 0x03, 0x02, 0x01, 0x01, 0x02, 0x01, 0x02}, &TestContextSpecificTags2{1, 2}}, + {[]byte{0x30, 0x03, 0x81, 0x01, '@'}, &TestContextSpecificTags3{"@"}}, {[]byte{0x01, 0x01, 0x00}, newBool(false)}, {[]byte{0x01, 0x01, 0xff}, newBool(true)}, {[]byte{0x30, 0x0b, 0x13, 0x03, 0x66, 0x6f, 0x6f, 0x02, 0x01, 0x22, 0x02, 0x01, 0x33}, &TestElementsAfterString{"foo", 0x22, 0x33}}, @@ -812,3 +817,51 @@ func TestStringSlice(t *testing.T) { } } } + +type explicitTaggedTimeTest struct { + Time time.Time `asn1:"explicit,tag:0"` +} + +var explicitTaggedTimeTestData = []struct { + in []byte + out explicitTaggedTimeTest +}{ + {[]byte{0x30, 0x11, 0xa0, 0xf, 0x17, 0xd, '9', '1', '0', '5', '0', '6', '1', '6', '4', '5', '4', '0', 'Z'}, + explicitTaggedTimeTest{time.Date(1991, 05, 06, 16, 45, 40, 0, time.UTC)}}, + {[]byte{0x30, 0x17, 0xa0, 0xf, 0x18, 0x13, '2', '0', '1', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '+', '0', '6', '0', '7'}, + explicitTaggedTimeTest{time.Date(2010, 01, 02, 03, 04, 05, 0, time.FixedZone("", 6*60*60+7*60))}}, +} + +func TestExplicitTaggedTime(t *testing.T) { + // Test that a time.Time will match either tagUTCTime or + // tagGeneralizedTime. + for i, test := range explicitTaggedTimeTestData { + var got explicitTaggedTimeTest + _, err := Unmarshal(test.in, &got) + if err != nil { + t.Errorf("Unmarshal failed at index %d %v", i, err) + } + if !got.Time.Equal(test.out.Time) { + t.Errorf("#%d: got %v, want %v", i, got.Time, test.out.Time) + } + } +} + +type implicitTaggedTimeTest struct { + Time time.Time `asn1:"tag:24"` +} + +func TestImplicitTaggedTime(t *testing.T) { + // An implicitly tagged time value, that happens to have an implicit + // tag equal to a GENERALIZEDTIME, should still be parsed as a UTCTime. + // (There's no "timeType" in fieldParameters to determine what type of + // time should be expected when implicitly tagged.) + der := []byte{0x30, 0x0f, 0x80 | 24, 0xd, '9', '1', '0', '5', '0', '6', '1', '6', '4', '5', '4', '0', 'Z'} + var result implicitTaggedTimeTest + if _, err := Unmarshal(der, &result); err != nil { + t.Fatalf("Error while parsing: %s", err) + } + if expected := time.Date(1991, 05, 06, 16, 45, 40, 0, time.UTC); !result.Time.Equal(expected) { + t.Errorf("Wrong result. Got %v, want %v", result.Time, expected) + } +} diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index e26fe59..b2f104b 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -513,8 +513,22 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) return } - if params.optional && reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) { - return + if params.optional && params.defaultValue != nil && canHaveDefaultValue(v.Kind()) { + defaultValue := reflect.New(v.Type()).Elem() + defaultValue.SetInt(*params.defaultValue) + + if reflect.DeepEqual(v.Interface(), defaultValue.Interface()) { + return + } + } + + // If no default value is given then the zero value for the type is + // assumed to be the default value. This isn't obviously the correct + // behaviour, but it's what Go has traditionally done. + if params.optional && params.defaultValue == nil { + if reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) { + return + } } if v.Type() == rawValueType { diff --git a/libgo/go/encoding/asn1/marshal_test.go b/libgo/go/encoding/asn1/marshal_test.go index a15acbe..5b0115f2 100644 --- a/libgo/go/encoding/asn1/marshal_test.go +++ b/libgo/go/encoding/asn1/marshal_test.go @@ -58,6 +58,10 @@ type omitEmptyTest struct { A []string `asn1:"omitempty"` } +type defaultTest struct { + A int `asn1:"optional,default:1"` +} + type testSET []int var PST = time.FixedZone("PST", -8*60*60) @@ -133,6 +137,9 @@ var marshalTests = []marshalTest{ {omitEmptyTest{[]string{}}, "3000"}, {omitEmptyTest{[]string{"1"}}, "30053003130131"}, {"Σ", "0c02cea3"}, + {defaultTest{0}, "3003020100"}, + {defaultTest{1}, "3000"}, + {defaultTest{2}, "3003020102"}, } func TestMarshal(t *testing.T) { diff --git a/libgo/go/encoding/base32/base32.go b/libgo/go/encoding/base32/base32.go index d770de3..5a9e869 100644 --- a/libgo/go/encoding/base32/base32.go +++ b/libgo/go/encoding/base32/base32.go @@ -73,45 +73,43 @@ func (enc *Encoding) Encode(dst, src []byte) { } for len(src) > 0 { - dst[0] = 0 - dst[1] = 0 - dst[2] = 0 - dst[3] = 0 - dst[4] = 0 - dst[5] = 0 - dst[6] = 0 - dst[7] = 0 + var b0, b1, b2, b3, b4, b5, b6, b7 byte // Unpack 8x 5-bit source blocks into a 5 byte // destination quantum switch len(src) { default: - dst[7] |= src[4] & 0x1F - dst[6] |= src[4] >> 5 + b7 = src[4] & 0x1F + b6 = src[4] >> 5 fallthrough case 4: - dst[6] |= (src[3] << 3) & 0x1F - dst[5] |= (src[3] >> 2) & 0x1F - dst[4] |= src[3] >> 7 + b6 |= (src[3] << 3) & 0x1F + b5 = (src[3] >> 2) & 0x1F + b4 = src[3] >> 7 fallthrough case 3: - dst[4] |= (src[2] << 1) & 0x1F - dst[3] |= (src[2] >> 4) & 0x1F + b4 |= (src[2] << 1) & 0x1F + b3 = (src[2] >> 4) & 0x1F fallthrough case 2: - dst[3] |= (src[1] << 4) & 0x1F - dst[2] |= (src[1] >> 1) & 0x1F - dst[1] |= (src[1] >> 6) & 0x1F + b3 |= (src[1] << 4) & 0x1F + b2 = (src[1] >> 1) & 0x1F + b1 = (src[1] >> 6) & 0x1F fallthrough case 1: - dst[1] |= (src[0] << 2) & 0x1F - dst[0] |= src[0] >> 3 + b1 |= (src[0] << 2) & 0x1F + b0 = src[0] >> 3 } // Encode 5-bit blocks using the base32 alphabet - for j := 0; j < 8; j++ { - dst[j] = enc.encode[dst[j]] - } + dst[0] = enc.encode[b0] + dst[1] = enc.encode[b1] + dst[2] = enc.encode[b2] + dst[3] = enc.encode[b3] + dst[4] = enc.encode[b4] + dst[5] = enc.encode[b5] + dst[6] = enc.encode[b6] + dst[7] = enc.encode[b7] // Pad the final quantum if len(src) < 5 { @@ -330,7 +328,7 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { func (enc *Encoding) DecodeString(s string) ([]byte, error) { s = strings.Map(removeNewlinesMapper, s) dbuf := make([]byte, enc.DecodedLen(len(s))) - n, err := enc.Decode(dbuf, []byte(s)) + n, _, err := enc.decode(dbuf, []byte(s)) return dbuf[:n], err } diff --git a/libgo/go/encoding/base32/base32_test.go b/libgo/go/encoding/base32/base32_test.go index f56b996..5a68f06 100644 --- a/libgo/go/encoding/base32/base32_test.go +++ b/libgo/go/encoding/base32/base32_test.go @@ -284,3 +284,19 @@ LNEBUWIIDFON2CA3DBMJXXE5LNFY== t.Error("Decoded results not equal") } } + +func BenchmarkEncodeToString(b *testing.B) { + data := make([]byte, 8192) + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + StdEncoding.EncodeToString(data) + } +} + +func BenchmarkDecodeString(b *testing.B) { + data := StdEncoding.EncodeToString(make([]byte, 8192)) + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + StdEncoding.DecodeString(data) + } +} diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go index e38c26d..ad3abe6 100644 --- a/libgo/go/encoding/base64/base64.go +++ b/libgo/go/encoding/base64/base64.go @@ -74,31 +74,29 @@ func (enc *Encoding) Encode(dst, src []byte) { } for len(src) > 0 { - dst[0] = 0 - dst[1] = 0 - dst[2] = 0 - dst[3] = 0 + var b0, b1, b2, b3 byte // Unpack 4x 6-bit source blocks into a 4 byte // destination quantum switch len(src) { default: - dst[3] |= src[2] & 0x3F - dst[2] |= src[2] >> 6 + b3 = src[2] & 0x3F + b2 = src[2] >> 6 fallthrough case 2: - dst[2] |= (src[1] << 2) & 0x3F - dst[1] |= src[1] >> 4 + b2 |= (src[1] << 2) & 0x3F + b1 = src[1] >> 4 fallthrough case 1: - dst[1] |= (src[0] << 4) & 0x3F - dst[0] |= src[0] >> 2 + b1 |= (src[0] << 4) & 0x3F + b0 = src[0] >> 2 } // Encode 6-bit blocks using the base64 alphabet - for j := 0; j < 4; j++ { - dst[j] = enc.encode[dst[j]] - } + dst[0] = enc.encode[b0] + dst[1] = enc.encode[b1] + dst[2] = enc.encode[b2] + dst[3] = enc.encode[b3] // Pad the final quantum if len(src) < 3 { @@ -295,7 +293,7 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { func (enc *Encoding) DecodeString(s string) ([]byte, error) { s = strings.Map(removeNewlinesMapper, s) dbuf := make([]byte, enc.DecodedLen(len(s))) - n, err := enc.Decode(dbuf, []byte(s)) + n, _, err := enc.decode(dbuf, []byte(s)) return dbuf[:n], err } diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go index a075194..7d199bf 100644 --- a/libgo/go/encoding/base64/base64_test.go +++ b/libgo/go/encoding/base64/base64_test.go @@ -342,3 +342,19 @@ func TestDecoderIssue7733(t *testing.T) { t.Errorf("DecodeString = %q; want abcd", s) } } + +func BenchmarkEncodeToString(b *testing.B) { + data := make([]byte, 8192) + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + StdEncoding.EncodeToString(data) + } +} + +func BenchmarkDecodeString(b *testing.B) { + data := StdEncoding.EncodeToString(make([]byte, 8192)) + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + StdEncoding.DecodeString(data) + } +} diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index a569487..466bf97 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -10,9 +10,10 @@ // type (int8, uint8, int16, float32, complex64, ...) // or an array or struct containing only fixed-size values. // -// Varints are a method of encoding integers using one or more bytes; -// numbers with smaller absolute value take a smaller number of bytes. -// For a specification, see http://code.google.com/apis/protocolbuffers/docs/encoding.html. +// The varint functions encode and decode single integer values using +// a variable-length encoding; smaller values require fewer bytes. +// For a specification, see +// http://code.google.com/apis/protocolbuffers/docs/encoding.html. // // This package favors simplicity over efficiency. Clients that require // high-performance serialization, especially for large data structures, @@ -199,18 +200,17 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error { } // Fallback to reflect-based decoding. - var v reflect.Value - switch d := reflect.ValueOf(data); d.Kind() { + v := reflect.ValueOf(data) + size := -1 + switch v.Kind() { case reflect.Ptr: - v = d.Elem() + v = v.Elem() + size = dataSize(v) case reflect.Slice: - v = d - default: - return errors.New("binary.Read: invalid type " + d.Type().String()) + size = dataSize(v) } - size, err := dataSize(v) - if err != nil { - return errors.New("binary.Read: " + err.Error()) + if size < 0 { + return errors.New("binary.Read: invalid type " + reflect.TypeOf(data).String()) } d := &decoder{order: order, buf: make([]byte, size)} if _, err := io.ReadFull(r, d.buf); err != nil { @@ -323,68 +323,64 @@ func Write(w io.Writer, order ByteOrder, data interface{}) error { // Fallback to reflect-based encoding. v := reflect.Indirect(reflect.ValueOf(data)) - size, err := dataSize(v) - if err != nil { - return errors.New("binary.Write: " + err.Error()) + size := dataSize(v) + if size < 0 { + return errors.New("binary.Write: invalid type " + reflect.TypeOf(data).String()) } buf := make([]byte, size) e := &encoder{order: order, buf: buf} e.value(v) - _, err = w.Write(buf) + _, err := w.Write(buf) return err } // Size returns how many bytes Write would generate to encode the value v, which // must be a fixed-size value or a slice of fixed-size values, or a pointer to such data. +// If v is neither of these, Size returns -1. func Size(v interface{}) int { - n, err := dataSize(reflect.Indirect(reflect.ValueOf(v))) - if err != nil { - return -1 - } - return n + return dataSize(reflect.Indirect(reflect.ValueOf(v))) } // dataSize returns the number of bytes the actual data represented by v occupies in memory. // For compound structures, it sums the sizes of the elements. Thus, for instance, for a slice // it returns the length of the slice times the element size and does not count the memory -// occupied by the header. -func dataSize(v reflect.Value) (int, error) { +// occupied by the header. If the type of v is not acceptable, dataSize returns -1. +func dataSize(v reflect.Value) int { if v.Kind() == reflect.Slice { - elem, err := sizeof(v.Type().Elem()) - if err != nil { - return 0, err + if s := sizeof(v.Type().Elem()); s >= 0 { + return s * v.Len() } - return v.Len() * elem, nil + return -1 } return sizeof(v.Type()) } -func sizeof(t reflect.Type) (int, error) { +// sizeof returns the size >= 0 of variables for the given type or -1 if the type is not acceptable. +func sizeof(t reflect.Type) int { switch t.Kind() { case reflect.Array: - n, err := sizeof(t.Elem()) - if err != nil { - return 0, err + if s := sizeof(t.Elem()); s >= 0 { + return s * t.Len() } - return t.Len() * n, nil case reflect.Struct: sum := 0 for i, n := 0, t.NumField(); i < n; i++ { - s, err := sizeof(t.Field(i).Type) - if err != nil { - return 0, err + s := sizeof(t.Field(i).Type) + if s < 0 { + return -1 } sum += s } - return sum, nil + return sum case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: - return int(t.Size()), nil + return int(t.Size()) } - return 0, errors.New("invalid type " + t.String()) + + return -1 } type coder struct { @@ -594,12 +590,11 @@ func (e *encoder) value(v reflect.Value) { } func (d *decoder) skip(v reflect.Value) { - n, _ := dataSize(v) - d.buf = d.buf[n:] + d.buf = d.buf[dataSize(v):] } func (e *encoder) skip(v reflect.Value) { - n, _ := dataSize(v) + n := dataSize(v) for i := range e.buf[0:n] { e.buf[i] = 0 } diff --git a/libgo/go/encoding/binary/binary_test.go b/libgo/go/encoding/binary/binary_test.go index c80c903..8ee595f 100644 --- a/libgo/go/encoding/binary/binary_test.go +++ b/libgo/go/encoding/binary/binary_test.go @@ -289,6 +289,26 @@ func TestUnexportedRead(t *testing.T) { Read(&buf, LittleEndian, &u2) } +func TestReadErrorMsg(t *testing.T) { + var buf bytes.Buffer + read := func(data interface{}) { + err := Read(&buf, LittleEndian, data) + want := "binary.Read: invalid type " + reflect.TypeOf(data).String() + if err == nil { + t.Errorf("%T: got no error; want %q", data, want) + return + } + if got := err.Error(); got != want { + t.Errorf("%T: got %q; want %q", data, got, want) + } + } + read(0) + s := new(struct{}) + read(&s) + p := &s + read(&p) +} + type byteSliceReader struct { remain []byte } @@ -315,8 +335,7 @@ func BenchmarkReadStruct(b *testing.B) { bsr := &byteSliceReader{} var buf bytes.Buffer Write(&buf, BigEndian, &s) - n, _ := dataSize(reflect.ValueOf(s)) - b.SetBytes(int64(n)) + b.SetBytes(int64(dataSize(reflect.ValueOf(s)))) t := s b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/libgo/go/encoding/csv/writer.go b/libgo/go/encoding/csv/writer.go index 1faecb6..17e7bb7 100644 --- a/libgo/go/encoding/csv/writer.go +++ b/libgo/go/encoding/csv/writer.go @@ -115,10 +115,22 @@ func (w *Writer) WriteAll(records [][]string) (err error) { } // fieldNeedsQuotes returns true if our field must be enclosed in quotes. -// Empty fields, files with a Comma, fields with a quote or newline, and +// Fields with a Comma, fields with a quote or newline, and // fields which start with a space must be enclosed in quotes. +// We used to quote empty strings, but we do not anymore (as of Go 1.4). +// The two representations should be equivalent, but Postgres distinguishes +// quoted vs non-quoted empty string during database imports, and it has +// an option to force the quoted behavior for non-quoted CSV but it has +// no option to force the non-quoted behavior for quoted CSV, making +// CSV with quoted empty strings strictly less useful. +// Not quoting the empty string also makes this package match the behavior +// of Microsoft Excel and Google Drive. +// For Postgres, quote the data termating string `\.`. func (w *Writer) fieldNeedsQuotes(field string) bool { - if len(field) == 0 || strings.IndexRune(field, w.Comma) >= 0 || strings.IndexAny(field, "\"\r\n") >= 0 { + if field == "" { + return false + } + if field == `\.` || strings.IndexRune(field, w.Comma) >= 0 || strings.IndexAny(field, "\"\r\n") >= 0 { return true } diff --git a/libgo/go/encoding/csv/writer_test.go b/libgo/go/encoding/csv/writer_test.go index 22b740c..8ddca0a 100644 --- a/libgo/go/encoding/csv/writer_test.go +++ b/libgo/go/encoding/csv/writer_test.go @@ -28,6 +28,17 @@ var writeTests = []struct { {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\r\ndef\"\r\n", UseCRLF: true}, {Input: [][]string{{"abc\rdef"}}, Output: "\"abcdef\"\r\n", UseCRLF: true}, {Input: [][]string{{"abc\rdef"}}, Output: "\"abc\rdef\"\n", UseCRLF: false}, + {Input: [][]string{{""}}, Output: "\n"}, + {Input: [][]string{{"", ""}}, Output: ",\n"}, + {Input: [][]string{{"", "", ""}}, Output: ",,\n"}, + {Input: [][]string{{"", "", "a"}}, Output: ",,a\n"}, + {Input: [][]string{{"", "a", ""}}, Output: ",a,\n"}, + {Input: [][]string{{"", "a", "a"}}, Output: ",a,a\n"}, + {Input: [][]string{{"a", "", ""}}, Output: "a,,\n"}, + {Input: [][]string{{"a", "", "a"}}, Output: "a,,a\n"}, + {Input: [][]string{{"a", "a", ""}}, Output: "a,a,\n"}, + {Input: [][]string{{"a", "a", "a"}}, Output: "a,a,a\n"}, + {Input: [][]string{{`\.`}}, Output: "\"\\.\"\n"}, } func TestWrite(t *testing.T) { diff --git a/libgo/go/encoding/gob/codec_test.go b/libgo/go/encoding/gob/codec_test.go index fa57f37..56a7298 100644 --- a/libgo/go/encoding/gob/codec_test.go +++ b/libgo/go/encoding/gob/codec_test.go @@ -14,7 +14,6 @@ import ( "strings" "testing" "time" - "unsafe" ) var doFuzzTests = flag.Bool("gob.fuzz", false, "run the fuzz tests, which are large and very slow") @@ -51,10 +50,16 @@ func testError(t *testing.T) { return } +func newDecBuffer(data []byte) *decBuffer { + return &decBuffer{ + data: data, + } +} + // Test basic encode/decode routines for unsigned integers func TestUintCodec(t *testing.T) { defer testError(t) - b := new(bytes.Buffer) + b := new(encBuffer) encState := newEncoderState(b) for _, tt := range encodeT { b.Reset() @@ -63,10 +68,10 @@ func TestUintCodec(t *testing.T) { t.Errorf("encodeUint: %#x encode: expected % x got % x", tt.x, tt.b, b.Bytes()) } } - decState := newDecodeState(b) for u := uint64(0); ; u = (u + 1) * 7 { b.Reset() encState.encodeUint(u) + decState := newDecodeState(newDecBuffer(b.Bytes())) v := decState.decodeUint() if u != v { t.Errorf("Encode/Decode: sent %#x received %#x", u, v) @@ -79,10 +84,10 @@ func TestUintCodec(t *testing.T) { func verifyInt(i int64, t *testing.T) { defer testError(t) - var b = new(bytes.Buffer) + var b = new(encBuffer) encState := newEncoderState(b) encState.encodeInt(i) - decState := newDecodeState(b) + decState := newDecodeState(newDecBuffer(b.Bytes())) decState.buf = make([]byte, 8) j := decState.decodeInt() if i != j { @@ -119,14 +124,14 @@ var complexResult = []byte{0x07, 0xFE, 0x31, 0x40, 0xFE, 0x33, 0x40} // The result of encoding "hello" with field number 7 var bytesResult = []byte{0x07, 0x05, 'h', 'e', 'l', 'l', 'o'} -func newDecodeState(buf *bytes.Buffer) *decoderState { +func newDecodeState(buf *decBuffer) *decoderState { d := new(decoderState) d.b = buf d.buf = make([]byte, uint64Size) return d } -func newEncoderState(b *bytes.Buffer) *encoderState { +func newEncoderState(b *encBuffer) *encoderState { b.Reset() state := &encoderState{enc: nil, b: b} state.fieldnum = -1 @@ -136,14 +141,14 @@ func newEncoderState(b *bytes.Buffer) *encoderState { // Test instruction execution for encoding. // Do not run the machine yet; instead do individual instructions crafted by hand. func TestScalarEncInstructions(t *testing.T) { - var b = new(bytes.Buffer) + var b = new(encBuffer) // bool { - data := struct{ a bool }{true} - instr := &encInstr{encBool, 6, 0, 0} + var data bool = true + instr := &encInstr{encBool, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(boolResult, b.Bytes()) { t.Errorf("bool enc instructions: expected % x got % x", boolResult, b.Bytes()) } @@ -152,10 +157,10 @@ func TestScalarEncInstructions(t *testing.T) { // int { b.Reset() - data := struct{ a int }{17} - instr := &encInstr{encInt, 6, 0, 0} + var data int = 17 + instr := &encInstr{encInt, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(signedResult, b.Bytes()) { t.Errorf("int enc instructions: expected % x got % x", signedResult, b.Bytes()) } @@ -164,10 +169,10 @@ func TestScalarEncInstructions(t *testing.T) { // uint { b.Reset() - data := struct{ a uint }{17} - instr := &encInstr{encUint, 6, 0, 0} + var data uint = 17 + instr := &encInstr{encUint, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(unsignedResult, b.Bytes()) { t.Errorf("uint enc instructions: expected % x got % x", unsignedResult, b.Bytes()) } @@ -176,10 +181,10 @@ func TestScalarEncInstructions(t *testing.T) { // int8 { b.Reset() - data := struct{ a int8 }{17} - instr := &encInstr{encInt8, 6, 0, 0} + var data int8 = 17 + instr := &encInstr{encInt, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(signedResult, b.Bytes()) { t.Errorf("int8 enc instructions: expected % x got % x", signedResult, b.Bytes()) } @@ -188,10 +193,10 @@ func TestScalarEncInstructions(t *testing.T) { // uint8 { b.Reset() - data := struct{ a uint8 }{17} - instr := &encInstr{encUint8, 6, 0, 0} + var data uint8 = 17 + instr := &encInstr{encUint, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(unsignedResult, b.Bytes()) { t.Errorf("uint8 enc instructions: expected % x got % x", unsignedResult, b.Bytes()) } @@ -200,10 +205,10 @@ func TestScalarEncInstructions(t *testing.T) { // int16 { b.Reset() - data := struct{ a int16 }{17} - instr := &encInstr{encInt16, 6, 0, 0} + var data int16 = 17 + instr := &encInstr{encInt, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(signedResult, b.Bytes()) { t.Errorf("int16 enc instructions: expected % x got % x", signedResult, b.Bytes()) } @@ -212,10 +217,10 @@ func TestScalarEncInstructions(t *testing.T) { // uint16 { b.Reset() - data := struct{ a uint16 }{17} - instr := &encInstr{encUint16, 6, 0, 0} + var data uint16 = 17 + instr := &encInstr{encUint, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(unsignedResult, b.Bytes()) { t.Errorf("uint16 enc instructions: expected % x got % x", unsignedResult, b.Bytes()) } @@ -224,10 +229,10 @@ func TestScalarEncInstructions(t *testing.T) { // int32 { b.Reset() - data := struct{ a int32 }{17} - instr := &encInstr{encInt32, 6, 0, 0} + var data int32 = 17 + instr := &encInstr{encInt, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(signedResult, b.Bytes()) { t.Errorf("int32 enc instructions: expected % x got % x", signedResult, b.Bytes()) } @@ -236,10 +241,10 @@ func TestScalarEncInstructions(t *testing.T) { // uint32 { b.Reset() - data := struct{ a uint32 }{17} - instr := &encInstr{encUint32, 6, 0, 0} + var data uint32 = 17 + instr := &encInstr{encUint, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(unsignedResult, b.Bytes()) { t.Errorf("uint32 enc instructions: expected % x got % x", unsignedResult, b.Bytes()) } @@ -248,10 +253,10 @@ func TestScalarEncInstructions(t *testing.T) { // int64 { b.Reset() - data := struct{ a int64 }{17} - instr := &encInstr{encInt64, 6, 0, 0} + var data int64 = 17 + instr := &encInstr{encInt, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(signedResult, b.Bytes()) { t.Errorf("int64 enc instructions: expected % x got % x", signedResult, b.Bytes()) } @@ -260,10 +265,10 @@ func TestScalarEncInstructions(t *testing.T) { // uint64 { b.Reset() - data := struct{ a uint64 }{17} - instr := &encInstr{encUint64, 6, 0, 0} + var data uint64 = 17 + instr := &encInstr{encUint, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(unsignedResult, b.Bytes()) { t.Errorf("uint64 enc instructions: expected % x got % x", unsignedResult, b.Bytes()) } @@ -272,10 +277,10 @@ func TestScalarEncInstructions(t *testing.T) { // float32 { b.Reset() - data := struct{ a float32 }{17} - instr := &encInstr{encFloat32, 6, 0, 0} + var data float32 = 17 + instr := &encInstr{encFloat, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(floatResult, b.Bytes()) { t.Errorf("float32 enc instructions: expected % x got % x", floatResult, b.Bytes()) } @@ -284,10 +289,10 @@ func TestScalarEncInstructions(t *testing.T) { // float64 { b.Reset() - data := struct{ a float64 }{17} - instr := &encInstr{encFloat64, 6, 0, 0} + var data float64 = 17 + instr := &encInstr{encFloat, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(floatResult, b.Bytes()) { t.Errorf("float64 enc instructions: expected % x got % x", floatResult, b.Bytes()) } @@ -296,10 +301,10 @@ func TestScalarEncInstructions(t *testing.T) { // bytes == []uint8 { b.Reset() - data := struct{ a []byte }{[]byte("hello")} - instr := &encInstr{encUint8Array, 6, 0, 0} + data := []byte("hello") + instr := &encInstr{encUint8Array, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(bytesResult, b.Bytes()) { t.Errorf("bytes enc instructions: expected % x got % x", bytesResult, b.Bytes()) } @@ -308,28 +313,28 @@ func TestScalarEncInstructions(t *testing.T) { // string { b.Reset() - data := struct{ a string }{"hello"} - instr := &encInstr{encString, 6, 0, 0} + var data string = "hello" + instr := &encInstr{encString, 6, nil, 0} state := newEncoderState(b) - instr.op(instr, state, unsafe.Pointer(&data)) + instr.op(instr, state, reflect.ValueOf(data)) if !bytes.Equal(bytesResult, b.Bytes()) { t.Errorf("string enc instructions: expected % x got % x", bytesResult, b.Bytes()) } } } -func execDec(typ string, instr *decInstr, state *decoderState, t *testing.T, p unsafe.Pointer) { +func execDec(typ string, instr *decInstr, state *decoderState, t *testing.T, value reflect.Value) { defer testError(t) v := int(state.decodeUint()) if v+state.fieldnum != 6 { t.Fatalf("decoding field number %d, got %d", 6, v+state.fieldnum) } - instr.op(instr, state, decIndirect(p, instr.indir)) + instr.op(instr, state, value.Elem()) state.fieldnum = 6 } func newDecodeStateFromData(data []byte) *decoderState { - b := bytes.NewBuffer(data) + b := newDecBuffer(data) state := newDecodeState(b) state.fieldnum = -1 return state @@ -342,234 +347,198 @@ func TestScalarDecInstructions(t *testing.T) { // bool { - var data struct { - a bool - } - instr := &decInstr{decBool, 6, 0, 0, ovfl} + var data bool + instr := &decInstr{decBool, 6, nil, ovfl} state := newDecodeStateFromData(boolResult) - execDec("bool", instr, state, t, unsafe.Pointer(&data)) - if data.a != true { - t.Errorf("bool a = %v not true", data.a) + execDec("bool", instr, state, t, reflect.ValueOf(&data)) + if data != true { + t.Errorf("bool a = %v not true", data) } } // int { - var data struct { - a int - } - instr := &decInstr{decOpTable[reflect.Int], 6, 0, 0, ovfl} + var data int + instr := &decInstr{decOpTable[reflect.Int], 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("int a = %v not 17", data.a) + execDec("int", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("int a = %v not 17", data) } } // uint { - var data struct { - a uint - } - instr := &decInstr{decOpTable[reflect.Uint], 6, 0, 0, ovfl} + var data uint + instr := &decInstr{decOpTable[reflect.Uint], 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("uint a = %v not 17", data.a) + execDec("uint", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("uint a = %v not 17", data) } } // int8 { - var data struct { - a int8 - } - instr := &decInstr{decInt8, 6, 0, 0, ovfl} + var data int8 + instr := &decInstr{decInt8, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int8", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("int8 a = %v not 17", data.a) + execDec("int8", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("int8 a = %v not 17", data) } } // uint8 { - var data struct { - a uint8 - } - instr := &decInstr{decUint8, 6, 0, 0, ovfl} + var data uint8 + instr := &decInstr{decUint8, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint8", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("uint8 a = %v not 17", data.a) + execDec("uint8", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("uint8 a = %v not 17", data) } } // int16 { - var data struct { - a int16 - } - instr := &decInstr{decInt16, 6, 0, 0, ovfl} + var data int16 + instr := &decInstr{decInt16, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int16", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("int16 a = %v not 17", data.a) + execDec("int16", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("int16 a = %v not 17", data) } } // uint16 { - var data struct { - a uint16 - } - instr := &decInstr{decUint16, 6, 0, 0, ovfl} + var data uint16 + instr := &decInstr{decUint16, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint16", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("uint16 a = %v not 17", data.a) + execDec("uint16", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("uint16 a = %v not 17", data) } } // int32 { - var data struct { - a int32 - } - instr := &decInstr{decInt32, 6, 0, 0, ovfl} + var data int32 + instr := &decInstr{decInt32, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int32", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("int32 a = %v not 17", data.a) + execDec("int32", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("int32 a = %v not 17", data) } } // uint32 { - var data struct { - a uint32 - } - instr := &decInstr{decUint32, 6, 0, 0, ovfl} + var data uint32 + instr := &decInstr{decUint32, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint32", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("uint32 a = %v not 17", data.a) + execDec("uint32", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("uint32 a = %v not 17", data) } } // uintptr { - var data struct { - a uintptr - } - instr := &decInstr{decOpTable[reflect.Uintptr], 6, 0, 0, ovfl} + var data uintptr + instr := &decInstr{decOpTable[reflect.Uintptr], 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uintptr", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("uintptr a = %v not 17", data.a) + execDec("uintptr", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("uintptr a = %v not 17", data) } } // int64 { - var data struct { - a int64 - } - instr := &decInstr{decInt64, 6, 0, 0, ovfl} + var data int64 + instr := &decInstr{decInt64, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int64", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("int64 a = %v not 17", data.a) + execDec("int64", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("int64 a = %v not 17", data) } } // uint64 { - var data struct { - a uint64 - } - instr := &decInstr{decUint64, 6, 0, 0, ovfl} + var data uint64 + instr := &decInstr{decUint64, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint64", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("uint64 a = %v not 17", data.a) + execDec("uint64", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("uint64 a = %v not 17", data) } } // float32 { - var data struct { - a float32 - } - instr := &decInstr{decFloat32, 6, 0, 0, ovfl} + var data float32 + instr := &decInstr{decFloat32, 6, nil, ovfl} state := newDecodeStateFromData(floatResult) - execDec("float32", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("float32 a = %v not 17", data.a) + execDec("float32", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("float32 a = %v not 17", data) } } // float64 { - var data struct { - a float64 - } - instr := &decInstr{decFloat64, 6, 0, 0, ovfl} + var data float64 + instr := &decInstr{decFloat64, 6, nil, ovfl} state := newDecodeStateFromData(floatResult) - execDec("float64", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17 { - t.Errorf("float64 a = %v not 17", data.a) + execDec("float64", instr, state, t, reflect.ValueOf(&data)) + if data != 17 { + t.Errorf("float64 a = %v not 17", data) } } // complex64 { - var data struct { - a complex64 - } - instr := &decInstr{decOpTable[reflect.Complex64], 6, 0, 0, ovfl} + var data complex64 + instr := &decInstr{decOpTable[reflect.Complex64], 6, nil, ovfl} state := newDecodeStateFromData(complexResult) - execDec("complex", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17+19i { - t.Errorf("complex a = %v not 17+19i", data.a) + execDec("complex", instr, state, t, reflect.ValueOf(&data)) + if data != 17+19i { + t.Errorf("complex a = %v not 17+19i", data) } } // complex128 { - var data struct { - a complex128 - } - instr := &decInstr{decOpTable[reflect.Complex128], 6, 0, 0, ovfl} + var data complex128 + instr := &decInstr{decOpTable[reflect.Complex128], 6, nil, ovfl} state := newDecodeStateFromData(complexResult) - execDec("complex", instr, state, t, unsafe.Pointer(&data)) - if data.a != 17+19i { - t.Errorf("complex a = %v not 17+19i", data.a) + execDec("complex", instr, state, t, reflect.ValueOf(&data)) + if data != 17+19i { + t.Errorf("complex a = %v not 17+19i", data) } } // bytes == []uint8 { - var data struct { - a []byte - } - instr := &decInstr{decUint8Slice, 6, 0, 0, ovfl} + var data []byte + instr := &decInstr{decUint8Slice, 6, nil, ovfl} state := newDecodeStateFromData(bytesResult) - execDec("bytes", instr, state, t, unsafe.Pointer(&data)) - if string(data.a) != "hello" { - t.Errorf(`bytes a = %q not "hello"`, string(data.a)) + execDec("bytes", instr, state, t, reflect.ValueOf(&data)) + if string(data) != "hello" { + t.Errorf(`bytes a = %q not "hello"`, string(data)) } } // string { - var data struct { - a string - } - instr := &decInstr{decString, 6, 0, 0, ovfl} + var data string + instr := &decInstr{decString, 6, nil, ovfl} state := newDecodeStateFromData(bytesResult) - execDec("bytes", instr, state, t, unsafe.Pointer(&data)) - if data.a != "hello" { - t.Errorf(`bytes a = %q not "hello"`, data.a) + execDec("bytes", instr, state, t, reflect.ValueOf(&data)) + if data != "hello" { + t.Errorf(`bytes a = %q not "hello"`, data) } } } diff --git a/libgo/go/encoding/gob/debug.go b/libgo/go/encoding/gob/debug.go index 6117eb0..536bbdb 100644 --- a/libgo/go/encoding/gob/debug.go +++ b/libgo/go/encoding/gob/debug.go @@ -306,7 +306,7 @@ func (deb *debugger) common() CommonType { // Id typeId id = deb.typeId() default: - errorf("corrupted CommonType") + errorf("corrupted CommonType, delta is %d fieldNum is %d", delta, fieldNum) } } return CommonType{name, id} @@ -598,11 +598,11 @@ func (deb *debugger) printBuiltin(indent tab, id typeId) { fmt.Fprintf(os.Stderr, "%s%d\n", indent, x) case tFloat: x := deb.uint64() - fmt.Fprintf(os.Stderr, "%s%g\n", indent, floatFromBits(x)) + fmt.Fprintf(os.Stderr, "%s%g\n", indent, float64FromBits(x)) case tComplex: r := deb.uint64() i := deb.uint64() - fmt.Fprintf(os.Stderr, "%s%g+%gi\n", indent, floatFromBits(r), floatFromBits(i)) + fmt.Fprintf(os.Stderr, "%s%g+%gi\n", indent, float64FromBits(r), float64FromBits(i)) case tBytes: x := int(deb.uint64()) b := make([]byte, x) diff --git a/libgo/go/encoding/gob/dec_helpers.go b/libgo/go/encoding/gob/dec_helpers.go new file mode 100644 index 0000000..a1b6766 --- /dev/null +++ b/libgo/go/encoding/gob/dec_helpers.go @@ -0,0 +1,468 @@ +// Created by decgen --output dec_helpers.go; DO NOT EDIT + +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gob + +import ( + "math" + "reflect" +) + +var decArrayHelper = map[reflect.Kind]decHelper{ + reflect.Bool: decBoolArray, + reflect.Complex64: decComplex64Array, + reflect.Complex128: decComplex128Array, + reflect.Float32: decFloat32Array, + reflect.Float64: decFloat64Array, + reflect.Int: decIntArray, + reflect.Int16: decInt16Array, + reflect.Int32: decInt32Array, + reflect.Int64: decInt64Array, + reflect.Int8: decInt8Array, + reflect.String: decStringArray, + reflect.Uint: decUintArray, + reflect.Uint16: decUint16Array, + reflect.Uint32: decUint32Array, + reflect.Uint64: decUint64Array, + reflect.Uintptr: decUintptrArray, +} + +var decSliceHelper = map[reflect.Kind]decHelper{ + reflect.Bool: decBoolSlice, + reflect.Complex64: decComplex64Slice, + reflect.Complex128: decComplex128Slice, + reflect.Float32: decFloat32Slice, + reflect.Float64: decFloat64Slice, + reflect.Int: decIntSlice, + reflect.Int16: decInt16Slice, + reflect.Int32: decInt32Slice, + reflect.Int64: decInt64Slice, + reflect.Int8: decInt8Slice, + reflect.String: decStringSlice, + reflect.Uint: decUintSlice, + reflect.Uint16: decUint16Slice, + reflect.Uint32: decUint32Slice, + reflect.Uint64: decUint64Slice, + reflect.Uintptr: decUintptrSlice, +} + +func decBoolArray(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decBoolSlice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decBoolSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]bool) + if !ok { + // It is kind bool but not type bool. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding bool array or slice: length exceeds input size (%d elements)", length) + } + slice[i] = state.decodeUint() != 0 + } + return true +} + +func decComplex64Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decComplex64Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decComplex64Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]complex64) + if !ok { + // It is kind complex64 but not type complex64. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding complex64 array or slice: length exceeds input size (%d elements)", length) + } + real := float32FromBits(state.decodeUint(), ovfl) + imag := float32FromBits(state.decodeUint(), ovfl) + slice[i] = complex(float32(real), float32(imag)) + } + return true +} + +func decComplex128Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decComplex128Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decComplex128Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]complex128) + if !ok { + // It is kind complex128 but not type complex128. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding complex128 array or slice: length exceeds input size (%d elements)", length) + } + real := float64FromBits(state.decodeUint()) + imag := float64FromBits(state.decodeUint()) + slice[i] = complex(real, imag) + } + return true +} + +func decFloat32Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decFloat32Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decFloat32Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]float32) + if !ok { + // It is kind float32 but not type float32. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding float32 array or slice: length exceeds input size (%d elements)", length) + } + slice[i] = float32(float32FromBits(state.decodeUint(), ovfl)) + } + return true +} + +func decFloat64Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decFloat64Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decFloat64Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]float64) + if !ok { + // It is kind float64 but not type float64. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding float64 array or slice: length exceeds input size (%d elements)", length) + } + slice[i] = float64FromBits(state.decodeUint()) + } + return true +} + +func decIntArray(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decIntSlice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decIntSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]int) + if !ok { + // It is kind int but not type int. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding int array or slice: length exceeds input size (%d elements)", length) + } + x := state.decodeInt() + // MinInt and MaxInt + if x < ^int64(^uint(0)>>1) || int64(^uint(0)>>1) < x { + error_(ovfl) + } + slice[i] = int(x) + } + return true +} + +func decInt16Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decInt16Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decInt16Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]int16) + if !ok { + // It is kind int16 but not type int16. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding int16 array or slice: length exceeds input size (%d elements)", length) + } + x := state.decodeInt() + if x < math.MinInt16 || math.MaxInt16 < x { + error_(ovfl) + } + slice[i] = int16(x) + } + return true +} + +func decInt32Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decInt32Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decInt32Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]int32) + if !ok { + // It is kind int32 but not type int32. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding int32 array or slice: length exceeds input size (%d elements)", length) + } + x := state.decodeInt() + if x < math.MinInt32 || math.MaxInt32 < x { + error_(ovfl) + } + slice[i] = int32(x) + } + return true +} + +func decInt64Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decInt64Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decInt64Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]int64) + if !ok { + // It is kind int64 but not type int64. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding int64 array or slice: length exceeds input size (%d elements)", length) + } + slice[i] = state.decodeInt() + } + return true +} + +func decInt8Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decInt8Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decInt8Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]int8) + if !ok { + // It is kind int8 but not type int8. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding int8 array or slice: length exceeds input size (%d elements)", length) + } + x := state.decodeInt() + if x < math.MinInt8 || math.MaxInt8 < x { + error_(ovfl) + } + slice[i] = int8(x) + } + return true +} + +func decStringArray(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decStringSlice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decStringSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]string) + if !ok { + // It is kind string but not type string. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding string array or slice: length exceeds input size (%d elements)", length) + } + u := state.decodeUint() + n := int(u) + if n < 0 || uint64(n) != u || n > state.b.Len() { + errorf("length of string exceeds input size (%d bytes)", u) + } + if n > state.b.Len() { + errorf("string data too long for buffer: %d", n) + } + // Read the data. + data := make([]byte, n) + if _, err := state.b.Read(data); err != nil { + errorf("error decoding string: %s", err) + } + slice[i] = string(data) + } + return true +} + +func decUintArray(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decUintSlice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decUintSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]uint) + if !ok { + // It is kind uint but not type uint. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding uint array or slice: length exceeds input size (%d elements)", length) + } + x := state.decodeUint() + /*TODO if math.MaxUint32 < x { + error_(ovfl) + }*/ + slice[i] = uint(x) + } + return true +} + +func decUint16Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decUint16Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decUint16Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]uint16) + if !ok { + // It is kind uint16 but not type uint16. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding uint16 array or slice: length exceeds input size (%d elements)", length) + } + x := state.decodeUint() + if math.MaxUint16 < x { + error_(ovfl) + } + slice[i] = uint16(x) + } + return true +} + +func decUint32Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decUint32Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decUint32Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]uint32) + if !ok { + // It is kind uint32 but not type uint32. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding uint32 array or slice: length exceeds input size (%d elements)", length) + } + x := state.decodeUint() + if math.MaxUint32 < x { + error_(ovfl) + } + slice[i] = uint32(x) + } + return true +} + +func decUint64Array(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decUint64Slice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decUint64Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]uint64) + if !ok { + // It is kind uint64 but not type uint64. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding uint64 array or slice: length exceeds input size (%d elements)", length) + } + slice[i] = state.decodeUint() + } + return true +} + +func decUintptrArray(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return decUintptrSlice(state, v.Slice(0, v.Len()), length, ovfl) +} + +func decUintptrSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]uintptr) + if !ok { + // It is kind uintptr but not type uintptr. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding uintptr array or slice: length exceeds input size (%d elements)", length) + } + x := state.decodeUint() + if uint64(^uintptr(0)) < x { + error_(ovfl) + } + slice[i] = uintptr(x) + } + return true +} diff --git a/libgo/go/encoding/gob/decgen.go b/libgo/go/encoding/gob/decgen.go new file mode 100644 index 0000000..da41a89 --- /dev/null +++ b/libgo/go/encoding/gob/decgen.go @@ -0,0 +1,240 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// encgen writes the helper functions for encoding. Intended to be +// used with go generate; see the invocation in encode.go. + +// TODO: We could do more by being unsafe. Add a -unsafe flag? + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "log" + "os" +) + +var output = flag.String("output", "dec_helpers.go", "file name to write") + +type Type struct { + lower string + upper string + decoder string +} + +var types = []Type{ + { + "bool", + "Bool", + `slice[i] = state.decodeUint() != 0`, + }, + { + "complex64", + "Complex64", + `real := float32FromBits(state.decodeUint(), ovfl) + imag := float32FromBits(state.decodeUint(), ovfl) + slice[i] = complex(float32(real), float32(imag))`, + }, + { + "complex128", + "Complex128", + `real := float64FromBits(state.decodeUint()) + imag := float64FromBits(state.decodeUint()) + slice[i] = complex(real, imag)`, + }, + { + "float32", + "Float32", + `slice[i] = float32(float32FromBits(state.decodeUint(), ovfl))`, + }, + { + "float64", + "Float64", + `slice[i] = float64FromBits(state.decodeUint())`, + }, + { + "int", + "Int", + `x := state.decodeInt() + // MinInt and MaxInt + if x < ^int64(^uint(0)>>1) || int64(^uint(0)>>1) < x { + error_(ovfl) + } + slice[i] = int(x)`, + }, + { + "int16", + "Int16", + `x := state.decodeInt() + if x < math.MinInt16 || math.MaxInt16 < x { + error_(ovfl) + } + slice[i] = int16(x)`, + }, + { + "int32", + "Int32", + `x := state.decodeInt() + if x < math.MinInt32 || math.MaxInt32 < x { + error_(ovfl) + } + slice[i] = int32(x)`, + }, + { + "int64", + "Int64", + `slice[i] = state.decodeInt()`, + }, + { + "int8", + "Int8", + `x := state.decodeInt() + if x < math.MinInt8 || math.MaxInt8 < x { + error_(ovfl) + } + slice[i] = int8(x)`, + }, + { + "string", + "String", + `u := state.decodeUint() + n := int(u) + if n < 0 || uint64(n) != u || n > state.b.Len() { + errorf("length of string exceeds input size (%d bytes)", u) + } + if n > state.b.Len() { + errorf("string data too long for buffer: %d", n) + } + // Read the data. + data := make([]byte, n) + if _, err := state.b.Read(data); err != nil { + errorf("error decoding string: %s", err) + } + slice[i] = string(data)`, + }, + { + "uint", + "Uint", + `x := state.decodeUint() + /*TODO if math.MaxUint32 < x { + error_(ovfl) + }*/ + slice[i] = uint(x)`, + }, + { + "uint16", + "Uint16", + `x := state.decodeUint() + if math.MaxUint16 < x { + error_(ovfl) + } + slice[i] = uint16(x)`, + }, + { + "uint32", + "Uint32", + `x := state.decodeUint() + if math.MaxUint32 < x { + error_(ovfl) + } + slice[i] = uint32(x)`, + }, + { + "uint64", + "Uint64", + `slice[i] = state.decodeUint()`, + }, + { + "uintptr", + "Uintptr", + `x := state.decodeUint() + if uint64(^uintptr(0)) < x { + error_(ovfl) + } + slice[i] = uintptr(x)`, + }, + // uint8 Handled separately. +} + +func main() { + log.SetFlags(0) + log.SetPrefix("decgen: ") + flag.Parse() + if flag.NArg() != 0 { + log.Fatal("usage: decgen [--output filename]") + } + var b bytes.Buffer + fmt.Fprintf(&b, "// Created by decgen --output %s; DO NOT EDIT\n", *output) + fmt.Fprint(&b, header) + printMaps(&b, "Array") + fmt.Fprint(&b, "\n") + printMaps(&b, "Slice") + for _, t := range types { + fmt.Fprintf(&b, arrayHelper, t.lower, t.upper) + fmt.Fprintf(&b, sliceHelper, t.lower, t.upper, t.decoder) + } + source, err := format.Source(b.Bytes()) + if err != nil { + log.Fatal("source format error:", err) + } + fd, err := os.Create(*output) + _, err = fd.Write(source) + if err != nil { + log.Fatal(err) + } +} + +func printMaps(b *bytes.Buffer, upperClass string) { + fmt.Fprintf(b, "var dec%sHelper = map[reflect.Kind]decHelper{\n", upperClass) + for _, t := range types { + fmt.Fprintf(b, "reflect.%s: dec%s%s,\n", t.upper, t.upper, upperClass) + } + fmt.Fprintf(b, "}\n") +} + +const header = ` +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gob + +import ( + "math" + "reflect" +) + +` + +const arrayHelper = ` +func dec%[2]sArray(state *decoderState, v reflect.Value, length int, ovfl error) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return dec%[2]sSlice(state, v.Slice(0, v.Len()), length, ovfl) +} +` + +const sliceHelper = ` +func dec%[2]sSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { + slice, ok := v.Interface().([]%[1]s) + if !ok { + // It is kind %[1]s but not type %[1]s. TODO: We can handle this unsafely. + return false + } + for i := 0; i < length; i++ { + if state.b.Len() == 0 { + errorf("decoding %[1]s array or slice: length exceeds input size (%%d elements)", length) + } + %[3]s + } + return true +} +` diff --git a/libgo/go/encoding/gob/decode.go b/libgo/go/encoding/gob/decode.go index d851314..a5bef93 100644 --- a/libgo/go/encoding/gob/decode.go +++ b/libgo/go/encoding/gob/decode.go @@ -2,19 +2,16 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package gob +//go:generate go run decgen.go -output dec_helpers.go -// TODO(rsc): When garbage collector changes, revisit -// the allocations in this file that use unsafe.Pointer. +package gob import ( - "bytes" "encoding" "errors" "io" "math" "reflect" - "unsafe" ) var ( @@ -23,21 +20,79 @@ var ( errRange = errors.New("gob: bad data: field numbers out of bounds") ) +type decHelper func(state *decoderState, v reflect.Value, length int, ovfl error) bool + // decoderState is the execution state of an instance of the decoder. A new state // is created for nested objects. type decoderState struct { dec *Decoder // The buffer is stored with an extra indirection because it may be replaced // if we load a type during decode (when reading an interface value). - b *bytes.Buffer + b *decBuffer fieldnum int // the last field number read. buf []byte next *decoderState // for free list } +// decBuffer is an extremely simple, fast implementation of a read-only byte buffer. +// It is initialized by calling Size and then copying the data into the slice returned by Bytes(). +type decBuffer struct { + data []byte + offset int // Read offset. +} + +func (d *decBuffer) Read(p []byte) (int, error) { + n := copy(p, d.data[d.offset:]) + if n == 0 && len(p) != 0 { + return 0, io.EOF + } + d.offset += n + return n, nil +} + +func (d *decBuffer) Drop(n int) { + if n > d.Len() { + panic("drop") + } + d.offset += n +} + +// Size grows the buffer to exactly n bytes, so d.Bytes() will +// return a slice of length n. Existing data is first discarded. +func (d *decBuffer) Size(n int) { + d.Reset() + if cap(d.data) < n { + d.data = make([]byte, n) + } else { + d.data = d.data[0:n] + } +} + +func (d *decBuffer) ReadByte() (byte, error) { + if d.offset >= len(d.data) { + return 0, io.EOF + } + c := d.data[d.offset] + d.offset++ + return c, nil +} + +func (d *decBuffer) Len() int { + return len(d.data) - d.offset +} + +func (d *decBuffer) Bytes() []byte { + return d.data[d.offset:] +} + +func (d *decBuffer) Reset() { + d.data = d.data[0:0] + d.offset = 0 +} + // We pass the bytes.Buffer separately for easier testing of the infrastructure // without requiring a full Decoder. -func (dec *Decoder) newDecoderState(buf *bytes.Buffer) *decoderState { +func (dec *Decoder) newDecoderState(buf *decBuffer) *decoderState { d := dec.freeList if d == nil { d = new(decoderState) @@ -128,175 +183,118 @@ func (state *decoderState) decodeInt() int64 { } // decOp is the signature of a decoding operator for a given type. -type decOp func(i *decInstr, state *decoderState, p unsafe.Pointer) +type decOp func(i *decInstr, state *decoderState, v reflect.Value) // The 'instructions' of the decoding machine type decInstr struct { - op decOp - field int // field number of the wire type - indir int // how many pointer indirections to reach the value in the struct - offset uintptr // offset in the structure of the field to encode - ovfl error // error message for overflow/underflow (for arrays, of the elements) -} - -// Since the encoder writes no zeros, if we arrive at a decoder we have -// a value to extract and store. The field number has already been read -// (it's how we knew to call this decoder). -// Each decoder is responsible for handling any indirections associated -// with the data structure. If any pointer so reached is nil, allocation must -// be done. - -// Walk the pointer hierarchy, allocating if we find a nil. Stop one before the end. -func decIndirect(p unsafe.Pointer, indir int) unsafe.Pointer { - for ; indir > 1; indir-- { - if *(*unsafe.Pointer)(p) == nil { - // Allocation required - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(unsafe.Pointer)) - } - p = *(*unsafe.Pointer)(p) - } - return p + op decOp + field int // field number of the wire type + index []int // field access indices for destination type + ovfl error // error message for overflow/underflow (for arrays, of the elements) } // ignoreUint discards a uint value with no destination. -func ignoreUint(i *decInstr, state *decoderState, p unsafe.Pointer) { +func ignoreUint(i *decInstr, state *decoderState, v reflect.Value) { state.decodeUint() } // ignoreTwoUints discards a uint value with no destination. It's used to skip // complex values. -func ignoreTwoUints(i *decInstr, state *decoderState, p unsafe.Pointer) { +func ignoreTwoUints(i *decInstr, state *decoderState, v reflect.Value) { state.decodeUint() state.decodeUint() } -// decBool decodes a uint and stores it as a boolean through p. -func decBool(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(bool)) +// Since the encoder writes no zeros, if we arrive at a decoder we have +// a value to extract and store. The field number has already been read +// (it's how we knew to call this decoder). +// Each decoder is responsible for handling any indirections associated +// with the data structure. If any pointer so reached is nil, allocation must +// be done. + +// decAlloc takes a value and returns a settable value that can +// be assigned to. If the value is a pointer, decAlloc guarantees it points to storage. +// The callers to the individual decoders are expected to have used decAlloc. +// The individual decoders don't need to it. +func decAlloc(v reflect.Value) reflect.Value { + for v.Kind() == reflect.Ptr { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) } - p = *(*unsafe.Pointer)(p) + v = v.Elem() } - *(*bool)(p) = state.decodeUint() != 0 + return v } -// decInt8 decodes an integer and stores it as an int8 through p. -func decInt8(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(int8)) - } - p = *(*unsafe.Pointer)(p) - } +// decBool decodes a uint and stores it as a boolean in value. +func decBool(i *decInstr, state *decoderState, value reflect.Value) { + value.SetBool(state.decodeUint() != 0) +} + +// decInt8 decodes an integer and stores it as an int8 in value. +func decInt8(i *decInstr, state *decoderState, value reflect.Value) { v := state.decodeInt() if v < math.MinInt8 || math.MaxInt8 < v { error_(i.ovfl) - } else { - *(*int8)(p) = int8(v) } + value.SetInt(v) } -// decUint8 decodes an unsigned integer and stores it as a uint8 through p. -func decUint8(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint8)) - } - p = *(*unsafe.Pointer)(p) - } +// decUint8 decodes an unsigned integer and stores it as a uint8 in value. +func decUint8(i *decInstr, state *decoderState, value reflect.Value) { v := state.decodeUint() if math.MaxUint8 < v { error_(i.ovfl) - } else { - *(*uint8)(p) = uint8(v) } + value.SetUint(v) } -// decInt16 decodes an integer and stores it as an int16 through p. -func decInt16(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(int16)) - } - p = *(*unsafe.Pointer)(p) - } +// decInt16 decodes an integer and stores it as an int16 in value. +func decInt16(i *decInstr, state *decoderState, value reflect.Value) { v := state.decodeInt() if v < math.MinInt16 || math.MaxInt16 < v { error_(i.ovfl) - } else { - *(*int16)(p) = int16(v) } + value.SetInt(v) } -// decUint16 decodes an unsigned integer and stores it as a uint16 through p. -func decUint16(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint16)) - } - p = *(*unsafe.Pointer)(p) - } +// decUint16 decodes an unsigned integer and stores it as a uint16 in value. +func decUint16(i *decInstr, state *decoderState, value reflect.Value) { v := state.decodeUint() if math.MaxUint16 < v { error_(i.ovfl) - } else { - *(*uint16)(p) = uint16(v) } + value.SetUint(v) } -// decInt32 decodes an integer and stores it as an int32 through p. -func decInt32(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(int32)) - } - p = *(*unsafe.Pointer)(p) - } +// decInt32 decodes an integer and stores it as an int32 in value. +func decInt32(i *decInstr, state *decoderState, value reflect.Value) { v := state.decodeInt() if v < math.MinInt32 || math.MaxInt32 < v { error_(i.ovfl) - } else { - *(*int32)(p) = int32(v) } + value.SetInt(v) } -// decUint32 decodes an unsigned integer and stores it as a uint32 through p. -func decUint32(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint32)) - } - p = *(*unsafe.Pointer)(p) - } +// decUint32 decodes an unsigned integer and stores it as a uint32 in value. +func decUint32(i *decInstr, state *decoderState, value reflect.Value) { v := state.decodeUint() if math.MaxUint32 < v { error_(i.ovfl) - } else { - *(*uint32)(p) = uint32(v) } + value.SetUint(v) } -// decInt64 decodes an integer and stores it as an int64 through p. -func decInt64(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(int64)) - } - p = *(*unsafe.Pointer)(p) - } - *(*int64)(p) = int64(state.decodeInt()) +// decInt64 decodes an integer and stores it as an int64 in value. +func decInt64(i *decInstr, state *decoderState, value reflect.Value) { + v := state.decodeInt() + value.SetInt(v) } -// decUint64 decodes an unsigned integer and stores it as a uint64 through p. -func decUint64(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(uint64)) - } - p = *(*unsafe.Pointer)(p) - } - *(*uint64)(p) = uint64(state.decodeUint()) +// decUint64 decodes an unsigned integer and stores it as a uint64 in value. +func decUint64(i *decInstr, state *decoderState, value reflect.Value) { + v := state.decodeUint() + value.SetUint(v) } // Floating-point numbers are transmitted as uint64s holding the bits @@ -304,7 +302,7 @@ func decUint64(i *decInstr, state *decoderState, p unsafe.Pointer) { // the exponent end coming out first, so integer floating point numbers // (for example) transmit more compactly. This routine does the // unswizzling. -func floatFromBits(u uint64) float64 { +func float64FromBits(u uint64) float64 { var v uint64 for i := 0; i < 8; i++ { v <<= 8 @@ -314,128 +312,100 @@ func floatFromBits(u uint64) float64 { return math.Float64frombits(v) } -// storeFloat32 decodes an unsigned integer, treats it as a 32-bit floating-point -// number, and stores it through p. It's a helper function for float32 and complex64. -func storeFloat32(i *decInstr, state *decoderState, p unsafe.Pointer) { - v := floatFromBits(state.decodeUint()) +// float32FromBits decodes an unsigned integer, treats it as a 32-bit floating-point +// number, and returns it. It's a helper function for float32 and complex64. +// It returns a float64 because that's what reflection needs, but its return +// value is known to be accurately representable in a float32. +func float32FromBits(u uint64, ovfl error) float64 { + v := float64FromBits(u) av := v if av < 0 { av = -av } // +Inf is OK in both 32- and 64-bit floats. Underflow is always OK. if math.MaxFloat32 < av && av <= math.MaxFloat64 { - error_(i.ovfl) - } else { - *(*float32)(p) = float32(v) + error_(ovfl) } + return v } // decFloat32 decodes an unsigned integer, treats it as a 32-bit floating-point -// number, and stores it through p. -func decFloat32(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(float32)) - } - p = *(*unsafe.Pointer)(p) - } - storeFloat32(i, state, p) +// number, and stores it in value. +func decFloat32(i *decInstr, state *decoderState, value reflect.Value) { + value.SetFloat(float32FromBits(state.decodeUint(), i.ovfl)) } // decFloat64 decodes an unsigned integer, treats it as a 64-bit floating-point -// number, and stores it through p. -func decFloat64(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(float64)) - } - p = *(*unsafe.Pointer)(p) - } - *(*float64)(p) = floatFromBits(uint64(state.decodeUint())) +// number, and stores it in value. +func decFloat64(i *decInstr, state *decoderState, value reflect.Value) { + value.SetFloat(float64FromBits(state.decodeUint())) } // decComplex64 decodes a pair of unsigned integers, treats them as a -// pair of floating point numbers, and stores them as a complex64 through p. +// pair of floating point numbers, and stores them as a complex64 in value. // The real part comes first. -func decComplex64(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex64)) - } - p = *(*unsafe.Pointer)(p) - } - storeFloat32(i, state, p) - storeFloat32(i, state, unsafe.Pointer(uintptr(p)+unsafe.Sizeof(float32(0)))) +func decComplex64(i *decInstr, state *decoderState, value reflect.Value) { + real := float32FromBits(state.decodeUint(), i.ovfl) + imag := float32FromBits(state.decodeUint(), i.ovfl) + value.SetComplex(complex(real, imag)) } // decComplex128 decodes a pair of unsigned integers, treats them as a -// pair of floating point numbers, and stores them as a complex128 through p. +// pair of floating point numbers, and stores them as a complex128 in value. // The real part comes first. -func decComplex128(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(complex128)) - } - p = *(*unsafe.Pointer)(p) - } - real := floatFromBits(uint64(state.decodeUint())) - imag := floatFromBits(uint64(state.decodeUint())) - *(*complex128)(p) = complex(real, imag) +func decComplex128(i *decInstr, state *decoderState, value reflect.Value) { + real := float64FromBits(state.decodeUint()) + imag := float64FromBits(state.decodeUint()) + value.SetComplex(complex(real, imag)) } -// decUint8Slice decodes a byte slice and stores through p a slice header +// decUint8Slice decodes a byte slice and stores in value a slice header // describing the data. // uint8 slices are encoded as an unsigned count followed by the raw bytes. -func decUint8Slice(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new([]uint8)) - } - p = *(*unsafe.Pointer)(p) +func decUint8Slice(i *decInstr, state *decoderState, value reflect.Value) { + u := state.decodeUint() + n := int(u) + if n < 0 || uint64(n) != u { + errorf("length of %s exceeds input size (%d bytes)", value.Type(), u) + } + if n > state.b.Len() { + errorf("%s data too long for buffer: %d", value.Type(), n) } - n := state.decodeUint() - if n > uint64(state.b.Len()) { - errorf("length of []byte exceeds input size (%d bytes)", n) + if n > tooBig { + errorf("byte slice too big: %d", n) } - slice := (*[]uint8)(p) - if uint64(cap(*slice)) < n { - *slice = make([]uint8, n) + if value.Cap() < n { + value.Set(reflect.MakeSlice(value.Type(), n, n)) } else { - *slice = (*slice)[0:n] + value.Set(value.Slice(0, n)) } - if _, err := state.b.Read(*slice); err != nil { + if _, err := state.b.Read(value.Bytes()); err != nil { errorf("error decoding []byte: %s", err) } } -// decString decodes byte array and stores through p a string header +// decString decodes byte array and stores in value a string header // describing the data. // Strings are encoded as an unsigned count followed by the raw bytes. -func decString(i *decInstr, state *decoderState, p unsafe.Pointer) { - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(new(string)) - } - p = *(*unsafe.Pointer)(p) +func decString(i *decInstr, state *decoderState, value reflect.Value) { + u := state.decodeUint() + n := int(u) + if n < 0 || uint64(n) != u || n > state.b.Len() { + errorf("length of %s exceeds input size (%d bytes)", value.Type(), u) } - n := state.decodeUint() - if n > uint64(state.b.Len()) { - errorf("string length exceeds input size (%d bytes)", n) + if n > state.b.Len() { + errorf("%s data too long for buffer: %d", value.Type(), n) } - b := make([]byte, n) - state.b.Read(b) - // It would be a shame to do the obvious thing here, - // *(*string)(p) = string(b) - // because we've already allocated the storage and this would - // allocate again and copy. So we do this ugly hack, which is even - // even more unsafe than it looks as it depends the memory - // representation of a string matching the beginning of the memory - // representation of a byte slice (a byte slice is longer). - *(*string)(p) = *(*string)(unsafe.Pointer(&b)) + // Read the data. + data := make([]byte, n) + if _, err := state.b.Read(data); err != nil { + errorf("error decoding string: %s", err) + } + value.SetString(string(data)) } // ignoreUint8Array skips over the data for a byte slice value with no destination. -func ignoreUint8Array(i *decInstr, state *decoderState, p unsafe.Pointer) { +func ignoreUint8Array(i *decInstr, state *decoderState, value reflect.Value) { b := make([]byte, state.decodeUint()) state.b.Read(b) } @@ -449,55 +419,29 @@ type decEngine struct { numInstr int // the number of active instructions } -// allocate makes sure storage is available for an object of underlying type rtyp -// that is indir levels of indirection through p. -func allocate(rtyp reflect.Type, p unsafe.Pointer, indir int) unsafe.Pointer { - if indir == 0 { - return p - } - up := p - if indir > 1 { - up = decIndirect(up, indir) - } - if *(*unsafe.Pointer)(up) == nil { - // Allocate object. - *(*unsafe.Pointer)(up) = unsafe.Pointer(reflect.New(rtyp).Pointer()) - } - return *(*unsafe.Pointer)(up) -} - -// decodeSingle decodes a top-level value that is not a struct and stores it through p. +// decodeSingle decodes a top-level value that is not a struct and stores it in value. // Such values are preceded by a zero, making them have the memory layout of a // struct field (although with an illegal field number). -func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, basep unsafe.Pointer) { +func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, value reflect.Value) { state := dec.newDecoderState(&dec.buf) + defer dec.freeDecoderState(state) state.fieldnum = singletonField - delta := int(state.decodeUint()) - if delta != 0 { + if state.decodeUint() != 0 { errorf("decode: corrupted data: non-zero delta for singleton") } instr := &engine.instr[singletonField] - if instr.indir != ut.indir { - errorf("internal error: inconsistent indirection instr %d ut %d", instr.indir, ut.indir) - } - ptr := basep // offset will be zero - if instr.indir > 1 { - ptr = decIndirect(ptr, instr.indir) - } - instr.op(instr, state, ptr) - dec.freeDecoderState(state) + instr.op(instr, state, value) } -// decodeStruct decodes a top-level struct and stores it through p. +// decodeStruct decodes a top-level struct and stores it in value. // Indir is for the value, not the type. At the time of the call it may // differ from ut.indir, which was computed when the engine was built. // This state cannot arise for decodeSingle, which is called directly // from the user's value, not from the innards of an engine. -func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, p unsafe.Pointer, indir int) { - p = allocate(ut.base, p, indir) +func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, value reflect.Value) { state := dec.newDecoderState(&dec.buf) + defer dec.freeDecoderState(state) state.fieldnum = -1 - basep := p for state.b.Len() > 0 { delta := int(state.decodeUint()) if delta < 0 { @@ -512,19 +456,25 @@ func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, p unsafe.P break } instr := &engine.instr[fieldnum] - p := unsafe.Pointer(uintptr(basep) + instr.offset) - if instr.indir > 1 { - p = decIndirect(p, instr.indir) + var field reflect.Value + if instr.index != nil { + // Otherwise the field is unknown to us and instr.op is an ignore op. + field = value.FieldByIndex(instr.index) + if field.Kind() == reflect.Ptr { + field = decAlloc(field) + } } - instr.op(instr, state, p) + instr.op(instr, state, field) state.fieldnum = fieldnum } - dec.freeDecoderState(state) } +var noValue reflect.Value + // ignoreStruct discards the data for a struct with no destination. func (dec *Decoder) ignoreStruct(engine *decEngine) { state := dec.newDecoderState(&dec.buf) + defer dec.freeDecoderState(state) state.fieldnum = -1 for state.b.Len() > 0 { delta := int(state.decodeUint()) @@ -539,97 +489,89 @@ func (dec *Decoder) ignoreStruct(engine *decEngine) { error_(errRange) } instr := &engine.instr[fieldnum] - instr.op(instr, state, unsafe.Pointer(nil)) + instr.op(instr, state, noValue) state.fieldnum = fieldnum } - dec.freeDecoderState(state) } // ignoreSingle discards the data for a top-level non-struct value with no // destination. It's used when calling Decode with a nil value. func (dec *Decoder) ignoreSingle(engine *decEngine) { state := dec.newDecoderState(&dec.buf) + defer dec.freeDecoderState(state) state.fieldnum = singletonField delta := int(state.decodeUint()) if delta != 0 { errorf("decode: corrupted data: non-zero delta for singleton") } instr := &engine.instr[singletonField] - instr.op(instr, state, unsafe.Pointer(nil)) - dec.freeDecoderState(state) + instr.op(instr, state, noValue) } // decodeArrayHelper does the work for decoding arrays and slices. -func (dec *Decoder) decodeArrayHelper(state *decoderState, p unsafe.Pointer, elemOp decOp, elemWid uintptr, length, elemIndir int, ovfl error) { - instr := &decInstr{elemOp, 0, elemIndir, 0, ovfl} +func (dec *Decoder) decodeArrayHelper(state *decoderState, value reflect.Value, elemOp decOp, length int, ovfl error, helper decHelper) { + if helper != nil && helper(state, value, length, ovfl) { + return + } + instr := &decInstr{elemOp, 0, nil, ovfl} + isPtr := value.Type().Elem().Kind() == reflect.Ptr for i := 0; i < length; i++ { if state.b.Len() == 0 { errorf("decoding array or slice: length exceeds input size (%d elements)", length) } - up := p - if elemIndir > 1 { - up = decIndirect(up, elemIndir) + v := value.Index(i) + if isPtr { + v = decAlloc(v) } - elemOp(instr, state, up) - p = unsafe.Pointer(uintptr(p) + elemWid) + elemOp(instr, state, v) } } -// decodeArray decodes an array and stores it through p, that is, p points to the zeroth element. +// decodeArray decodes an array and stores it in value. // The length is an unsigned integer preceding the elements. Even though the length is redundant // (it's part of the type), it's a useful check and is included in the encoding. -func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, p unsafe.Pointer, elemOp decOp, elemWid uintptr, length, indir, elemIndir int, ovfl error) { - if indir > 0 { - p = allocate(atyp, p, 1) // All but the last level has been allocated by dec.Indirect - } +func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, value reflect.Value, elemOp decOp, length int, ovfl error, helper decHelper) { if n := state.decodeUint(); n != uint64(length) { errorf("length mismatch in decodeArray") } - dec.decodeArrayHelper(state, p, elemOp, elemWid, length, elemIndir, ovfl) + dec.decodeArrayHelper(state, value, elemOp, length, ovfl, helper) } -// decodeIntoValue is a helper for map decoding. Since maps are decoded using reflection, -// unlike the other items we can't use a pointer directly. -func decodeIntoValue(state *decoderState, op decOp, indir int, v reflect.Value, ovfl error) reflect.Value { - instr := &decInstr{op, 0, indir, 0, ovfl} - up := unsafeAddr(v) - if indir > 1 { - up = decIndirect(up, indir) +// decodeIntoValue is a helper for map decoding. +func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Value, ovfl error) reflect.Value { + instr := &decInstr{op, 0, nil, ovfl} + v := value + if isPtr { + v = decAlloc(value) } - op(instr, state, up) - return v + op(instr, state, v) + return value } -// decodeMap decodes a map and stores its header through p. +// decodeMap decodes a map and stores it in value. // Maps are encoded as a length followed by key:value pairs. // Because the internals of maps are not visible to us, we must // use reflection rather than pointer magic. -func (dec *Decoder) decodeMap(mtyp reflect.Type, state *decoderState, p unsafe.Pointer, keyOp, elemOp decOp, indir, keyIndir, elemIndir int, ovfl error) { - if indir > 0 { - p = allocate(mtyp, p, 1) // All but the last level has been allocated by dec.Indirect - } - up := unsafe.Pointer(p) - if *(*unsafe.Pointer)(up) == nil { // maps are represented as a pointer in the runtime +func (dec *Decoder) decodeMap(mtyp reflect.Type, state *decoderState, value reflect.Value, keyOp, elemOp decOp, ovfl error) { + if value.IsNil() { // Allocate map. - *(*unsafe.Pointer)(up) = unsafe.Pointer(reflect.MakeMap(mtyp).Pointer()) + value.Set(reflect.MakeMap(mtyp)) } - // Maps cannot be accessed by moving addresses around the way - // that slices etc. can. We must recover a full reflection value for - // the iteration. - v := reflect.NewAt(mtyp, unsafe.Pointer(p)).Elem() n := int(state.decodeUint()) + keyIsPtr := mtyp.Key().Kind() == reflect.Ptr + elemIsPtr := mtyp.Elem().Kind() == reflect.Ptr for i := 0; i < n; i++ { - key := decodeIntoValue(state, keyOp, keyIndir, allocValue(mtyp.Key()), ovfl) - elem := decodeIntoValue(state, elemOp, elemIndir, allocValue(mtyp.Elem()), ovfl) - v.SetMapIndex(key, elem) + key := decodeIntoValue(state, keyOp, keyIsPtr, allocValue(mtyp.Key()), ovfl) + elem := decodeIntoValue(state, elemOp, elemIsPtr, allocValue(mtyp.Elem()), ovfl) + value.SetMapIndex(key, elem) } } // ignoreArrayHelper does the work for discarding arrays and slices. func (dec *Decoder) ignoreArrayHelper(state *decoderState, elemOp decOp, length int) { - instr := &decInstr{elemOp, 0, 0, 0, errors.New("no error")} + instr := &decInstr{elemOp, 0, nil, errors.New("no error")} for i := 0; i < length; i++ { - elemOp(instr, state, nil) + elemOp(instr, state, noValue) } } @@ -644,36 +586,34 @@ func (dec *Decoder) ignoreArray(state *decoderState, elemOp decOp, length int) { // ignoreMap discards the data for a map value with no destination. func (dec *Decoder) ignoreMap(state *decoderState, keyOp, elemOp decOp) { n := int(state.decodeUint()) - keyInstr := &decInstr{keyOp, 0, 0, 0, errors.New("no error")} - elemInstr := &decInstr{elemOp, 0, 0, 0, errors.New("no error")} + keyInstr := &decInstr{keyOp, 0, nil, errors.New("no error")} + elemInstr := &decInstr{elemOp, 0, nil, errors.New("no error")} for i := 0; i < n; i++ { - keyOp(keyInstr, state, nil) - elemOp(elemInstr, state, nil) + keyOp(keyInstr, state, noValue) + elemOp(elemInstr, state, noValue) } } -// decodeSlice decodes a slice and stores the slice header through p. +// decodeSlice decodes a slice and stores it in value. // Slices are encoded as an unsigned length followed by the elements. -func (dec *Decoder) decodeSlice(atyp reflect.Type, state *decoderState, p unsafe.Pointer, elemOp decOp, elemWid uintptr, indir, elemIndir int, ovfl error) { - nr := state.decodeUint() - n := int(nr) - if indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - // Allocate the slice header. - *(*unsafe.Pointer)(p) = unsafe.Pointer(new([]unsafe.Pointer)) - } - p = *(*unsafe.Pointer)(p) - } - // Allocate storage for the slice elements, that is, the underlying array, - // if the existing slice does not have the capacity. - // Always write a header at p. - hdrp := (*reflect.SliceHeader)(p) - if hdrp.Cap < n { - hdrp.Data = reflect.MakeSlice(atyp, n, n).Pointer() - hdrp.Cap = n +func (dec *Decoder) decodeSlice(state *decoderState, value reflect.Value, elemOp decOp, ovfl error, helper decHelper) { + u := state.decodeUint() + typ := value.Type() + size := uint64(typ.Elem().Size()) + nBytes := u * size + n := int(u) + // Take care with overflow in this calculation. + if n < 0 || uint64(n) != u || nBytes > tooBig || (size > 0 && nBytes/size != u) { + // We don't check n against buffer length here because if it's a slice + // of interfaces, there will be buffer reloads. + errorf("%s slice too big: %d elements of %d bytes", typ.Elem(), u, size) + } + if value.Cap() < n { + value.Set(reflect.MakeSlice(typ, n, n)) + } else { + value.Set(value.Slice(0, n)) } - hdrp.Len = n - dec.decodeArrayHelper(state, unsafe.Pointer(hdrp.Data), elemOp, elemWid, n, elemIndir, ovfl) + dec.decodeArrayHelper(state, value, elemOp, n, ovfl, helper) } // ignoreSlice skips over the data for a slice value with no destination. @@ -681,21 +621,10 @@ func (dec *Decoder) ignoreSlice(state *decoderState, elemOp decOp) { dec.ignoreArrayHelper(state, elemOp, int(state.decodeUint())) } -// setInterfaceValue sets an interface value to a concrete value, -// but first it checks that the assignment will succeed. -func setInterfaceValue(ivalue reflect.Value, value reflect.Value) { - if !value.Type().AssignableTo(ivalue.Type()) { - errorf("%s is not assignable to type %s", value.Type(), ivalue.Type()) - } - ivalue.Set(value) -} - -// decodeInterface decodes an interface value and stores it through p. +// decodeInterface decodes an interface value and stores it in value. // Interfaces are encoded as the name of a concrete type followed by a value. // If the name is empty, the value is nil and no value is sent. -func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, p unsafe.Pointer, indir int) { - // Create a writable interface reflect.Value. We need one even for the nil case. - ivalue := allocValue(ityp) +func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, value reflect.Value) { // Read the name of the concrete type. nr := state.decodeUint() if nr < 0 || nr > 1<<31 { // zero is permissible for anonymous types @@ -707,13 +636,10 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, p un b := make([]byte, nr) state.b.Read(b) name := string(b) + // Allocate the destination interface value. if name == "" { - // Copy the representation of the nil interface value to the target. - // This is horribly unsafe and special. - if indir > 0 { - p = allocate(ityp, p, 1) // All but the last level has been allocated by dec.Indirect - } - *(*[2]uintptr)(unsafe.Pointer(p)) = ivalue.InterfaceData() + // Copy the nil interface value to the target. + value.Set(reflect.Zero(value.Type())) return } if len(name) > 1024 { @@ -735,21 +661,18 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, p un // in case we want to ignore the value by skipping it completely). state.decodeUint() // Read the concrete value. - value := allocValue(typ) - dec.decodeValue(concreteId, value) + v := allocValue(typ) + dec.decodeValue(concreteId, v) if dec.err != nil { error_(dec.err) } - // Allocate the destination interface value. - if indir > 0 { - p = allocate(ityp, p, 1) // All but the last level has been allocated by dec.Indirect - } // Assign the concrete value to the interface. // Tread carefully; it might not satisfy the interface. - setInterfaceValue(ivalue, value) - // Copy the representation of the interface value to the target. - // This is horribly unsafe and special. - *(*[2]uintptr)(unsafe.Pointer(p)) = ivalue.InterfaceData() + if !typ.AssignableTo(ityp) { + errorf("%s is not assignable to type %s", typ, ityp) + } + // Copy the interface value to the target. + value.Set(v) } // ignoreInterface discards the data for an interface value with no destination. @@ -765,12 +688,12 @@ func (dec *Decoder) ignoreInterface(state *decoderState) { error_(dec.err) } // At this point, the decoder buffer contains a delimited value. Just toss it. - state.b.Next(int(state.decodeUint())) + state.b.Drop(int(state.decodeUint())) } // decodeGobDecoder decodes something implementing the GobDecoder interface. // The data is encoded as a byte slice. -func (dec *Decoder) decodeGobDecoder(ut *userTypeInfo, state *decoderState, v reflect.Value) { +func (dec *Decoder) decodeGobDecoder(ut *userTypeInfo, state *decoderState, value reflect.Value) { // Read the bytes for the value. b := make([]byte, state.decodeUint()) _, err := state.b.Read(b) @@ -780,11 +703,11 @@ func (dec *Decoder) decodeGobDecoder(ut *userTypeInfo, state *decoderState, v re // We know it's one of these. switch ut.externalDec { case xGob: - err = v.Interface().(GobDecoder).GobDecode(b) + err = value.Interface().(GobDecoder).GobDecode(b) case xBinary: - err = v.Interface().(encoding.BinaryUnmarshaler).UnmarshalBinary(b) + err = value.Interface().(encoding.BinaryUnmarshaler).UnmarshalBinary(b) case xText: - err = v.Interface().(encoding.TextUnmarshaler).UnmarshalText(b) + err = value.Interface().(encoding.TextUnmarshaler).UnmarshalText(b) } if err != nil { error_(err) @@ -832,7 +755,7 @@ var decIgnoreOpMap = map[typeId]decOp{ // decOpFor returns the decoding op for the base type under rt and // the indirection count to reach it. -func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProgress map[reflect.Type]*decOp) (*decOp, int) { +func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProgress map[reflect.Type]*decOp) *decOp { ut := userType(rt) // If the type implements GobEncoder, we handle it without further processing. if ut.externalDec != 0 { @@ -842,10 +765,9 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg // If this type is already in progress, it's a recursive type (e.g. map[string]*T). // Return the pointer to the op we're already building. if opPtr := inProgress[rt]; opPtr != nil { - return opPtr, ut.indir + return opPtr } typ := ut.base - indir := ut.indir var op decOp k := typ.Kind() if int(k) < len(decOpTable) { @@ -858,20 +780,21 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg case reflect.Array: name = "element of " + name elemId := dec.wireType[wireId].ArrayT.Elem - elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress) + elemOp := dec.decOpFor(elemId, t.Elem(), name, inProgress) ovfl := overflow(name) - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { - state.dec.decodeArray(t, state, p, *elemOp, t.Elem().Size(), t.Len(), i.indir, elemIndir, ovfl) + helper := decArrayHelper[t.Elem().Kind()] + op = func(i *decInstr, state *decoderState, value reflect.Value) { + state.dec.decodeArray(t, state, value, *elemOp, t.Len(), ovfl, helper) } case reflect.Map: keyId := dec.wireType[wireId].MapT.Key elemId := dec.wireType[wireId].MapT.Elem - keyOp, keyIndir := dec.decOpFor(keyId, t.Key(), "key of "+name, inProgress) - elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), "element of "+name, inProgress) + keyOp := dec.decOpFor(keyId, t.Key(), "key of "+name, inProgress) + elemOp := dec.decOpFor(elemId, t.Elem(), "element of "+name, inProgress) ovfl := overflow(name) - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { - state.dec.decodeMap(t, state, p, *keyOp, *elemOp, i.indir, keyIndir, elemIndir, ovfl) + op = func(i *decInstr, state *decoderState, value reflect.Value) { + state.dec.decodeMap(t, state, value, *keyOp, *elemOp, ovfl) } case reflect.Slice: @@ -886,32 +809,34 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg } else { elemId = dec.wireType[wireId].SliceT.Elem } - elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress) + elemOp := dec.decOpFor(elemId, t.Elem(), name, inProgress) ovfl := overflow(name) - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { - state.dec.decodeSlice(t, state, p, *elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl) + helper := decSliceHelper[t.Elem().Kind()] + op = func(i *decInstr, state *decoderState, value reflect.Value) { + state.dec.decodeSlice(state, value, *elemOp, ovfl, helper) } case reflect.Struct: // Generate a closure that calls out to the engine for the nested type. - enginePtr, err := dec.getDecEnginePtr(wireId, userType(typ)) + ut := userType(typ) + enginePtr, err := dec.getDecEnginePtr(wireId, ut) if err != nil { error_(err) } - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, value reflect.Value) { // indirect through enginePtr to delay evaluation for recursive structs. - dec.decodeStruct(*enginePtr, userType(typ), p, i.indir) + dec.decodeStruct(*enginePtr, ut, value) } case reflect.Interface: - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { - state.dec.decodeInterface(t, state, p, i.indir) + op = func(i *decInstr, state *decoderState, value reflect.Value) { + state.dec.decodeInterface(t, state, value) } } } if op == nil { errorf("decode can't handle type %s", rt) } - return &op, indir + return &op } // decIgnoreOpFor returns the decoding op for a field that has no destination. @@ -921,7 +846,7 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { if wireId == tInterface { // Special case because it's a method: the ignored item might // define types and we need to record their state in the decoder. - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, value reflect.Value) { state.dec.ignoreInterface(state) } return op @@ -934,7 +859,7 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { case wire.ArrayT != nil: elemId := wire.ArrayT.Elem elemOp := dec.decIgnoreOpFor(elemId) - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, value reflect.Value) { state.dec.ignoreArray(state, elemOp, wire.ArrayT.Len) } @@ -943,14 +868,14 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { elemId := dec.wireType[wireId].MapT.Elem keyOp := dec.decIgnoreOpFor(keyId) elemOp := dec.decIgnoreOpFor(elemId) - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, value reflect.Value) { state.dec.ignoreMap(state, keyOp, elemOp) } case wire.SliceT != nil: elemId := wire.SliceT.Elem elemOp := dec.decIgnoreOpFor(elemId) - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, value reflect.Value) { state.dec.ignoreSlice(state, elemOp) } @@ -960,13 +885,13 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { if err != nil { error_(err) } - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, value reflect.Value) { // indirect through enginePtr to delay evaluation for recursive structs state.dec.ignoreStruct(*enginePtr) } case wire.GobEncoderT != nil, wire.BinaryMarshalerT != nil, wire.TextMarshalerT != nil: - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { + op = func(i *decInstr, state *decoderState, value reflect.Value) { state.dec.ignoreGobDecoder(state) } } @@ -979,7 +904,7 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId) decOp { // gobDecodeOpFor returns the op for a type that is known to implement // GobDecoder. -func (dec *Decoder) gobDecodeOpFor(ut *userTypeInfo) (*decOp, int) { +func (dec *Decoder) gobDecodeOpFor(ut *userTypeInfo) *decOp { rcvrType := ut.user if ut.decIndir == -1 { rcvrType = reflect.PtrTo(rcvrType) @@ -989,25 +914,14 @@ func (dec *Decoder) gobDecodeOpFor(ut *userTypeInfo) (*decOp, int) { } } var op decOp - op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { - // Caller has gotten us to within one indirection of our value. - if i.indir > 0 { - if *(*unsafe.Pointer)(p) == nil { - *(*unsafe.Pointer)(p) = unsafe.Pointer(reflect.New(ut.base).Pointer()) - } + op = func(i *decInstr, state *decoderState, value reflect.Value) { + // We now have the base type. We need its address if the receiver is a pointer. + if value.Kind() != reflect.Ptr && rcvrType.Kind() == reflect.Ptr { + value = value.Addr() } - // Now p is a pointer to the base type. Do we need to climb out to - // get to the receiver type? - var v reflect.Value - if ut.decIndir == -1 { - v = reflect.NewAt(rcvrType, unsafe.Pointer(&p)).Elem() - } else { - v = reflect.NewAt(rcvrType, p).Elem() - } - state.dec.decodeGobDecoder(ut, state, v) + state.dec.decodeGobDecoder(ut, state, value) } - return &op, int(ut.indir) - + return &op } // compatibleType asks: Are these two gob Types compatible? @@ -1108,9 +1022,9 @@ func (dec *Decoder) compileSingle(remoteId typeId, ut *userTypeInfo) (engine *de } return nil, errors.New("gob: decoding into local type " + name + ", received remote type " + remoteType) } - op, indir := dec.decOpFor(remoteId, rt, name, make(map[reflect.Type]*decOp)) + op := dec.decOpFor(remoteId, rt, name, make(map[reflect.Type]*decOp)) ovfl := errors.New(`value for "` + name + `" out of range`) - engine.instr[singletonField] = decInstr{*op, singletonField, indir, 0, ovfl} + engine.instr[singletonField] = decInstr{*op, singletonField, nil, ovfl} engine.numInstr = 1 return } @@ -1121,7 +1035,7 @@ func (dec *Decoder) compileIgnoreSingle(remoteId typeId) (engine *decEngine, err engine.instr = make([]decInstr, 1) // one item op := dec.decIgnoreOpFor(remoteId) ovfl := overflow(dec.typeString(remoteId)) - engine.instr[0] = decInstr{op, 0, 0, 0, ovfl} + engine.instr[0] = decInstr{op, 0, nil, ovfl} engine.numInstr = 1 return } @@ -1164,14 +1078,14 @@ func (dec *Decoder) compileDec(remoteId typeId, ut *userTypeInfo) (engine *decEn // TODO(r): anonymous names if !present || !isExported(wireField.Name) { op := dec.decIgnoreOpFor(wireField.Id) - engine.instr[fieldnum] = decInstr{op, fieldnum, 0, 0, ovfl} + engine.instr[fieldnum] = decInstr{op, fieldnum, nil, ovfl} continue } if !dec.compatibleType(localField.Type, wireField.Id, make(map[reflect.Type]typeId)) { errorf("wrong type (%s) for received field %s.%s", localField.Type, wireStruct.Name, wireField.Name) } - op, indir := dec.decOpFor(wireField.Id, localField.Type, localField.Name, seen) - engine.instr[fieldnum] = decInstr{*op, fieldnum, indir, uintptr(localField.Offset), ovfl} + op := dec.decOpFor(wireField.Id, localField.Type, localField.Name, seen) + engine.instr[fieldnum] = decInstr{*op, fieldnum, localField.Index, ovfl} engine.numInstr++ } return @@ -1222,22 +1136,23 @@ func (dec *Decoder) getIgnoreEnginePtr(wireId typeId) (enginePtr **decEngine, er return } -// decodeValue decodes the data stream representing a value and stores it in val. -func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) { +// decodeValue decodes the data stream representing a value and stores it in value. +func (dec *Decoder) decodeValue(wireId typeId, value reflect.Value) { defer catchError(&dec.err) // If the value is nil, it means we should just ignore this item. - if !val.IsValid() { + if !value.IsValid() { dec.decodeIgnoredValue(wireId) return } // Dereference down to the underlying type. - ut := userType(val.Type()) + ut := userType(value.Type()) base := ut.base var enginePtr **decEngine enginePtr, dec.err = dec.getDecEnginePtr(wireId, ut) if dec.err != nil { return } + value = decAlloc(value) engine := *enginePtr if st := base; st.Kind() == reflect.Struct && ut.externalDec == 0 { if engine.numInstr == 0 && st.NumField() > 0 && @@ -1245,9 +1160,9 @@ func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) { name := base.Name() errorf("type mismatch: no fields matched compiling decoder for %s", name) } - dec.decodeStruct(engine, ut, unsafeAddr(val), ut.indir) + dec.decodeStruct(engine, ut, value) } else { - dec.decodeSingle(engine, ut, unsafeAddr(val)) + dec.decodeSingle(engine, ut, value) } } @@ -1293,21 +1208,6 @@ func init() { decOpTable[reflect.Uintptr] = uop } -// Gob assumes it can call UnsafeAddr on any Value -// in order to get a pointer it can copy data from. -// Values that have just been created and do not point -// into existing structs or slices cannot be addressed, -// so simulate it by returning a pointer to a copy. -// Each call allocates once. -func unsafeAddr(v reflect.Value) unsafe.Pointer { - if v.CanAddr() { - return unsafe.Pointer(v.UnsafeAddr()) - } - x := reflect.New(v.Type()).Elem() - x.Set(v) - return unsafe.Pointer(x.UnsafeAddr()) -} - // Gob depends on being able to take the address // of zeroed Values it creates, so use this wrapper instead // of the standard reflect.Zero. diff --git a/libgo/go/encoding/gob/decoder.go b/libgo/go/encoding/gob/decoder.go index 3a769ec..c453e9b 100644 --- a/libgo/go/encoding/gob/decoder.go +++ b/libgo/go/encoding/gob/decoder.go @@ -6,25 +6,28 @@ package gob import ( "bufio" - "bytes" "errors" "io" "reflect" "sync" ) +// tooBig provides a sanity check for sizes; used in several places. +// Upper limit of 1GB, allowing room to grow a little without overflow. +// TODO: make this adjustable? +const tooBig = 1 << 30 + // A Decoder manages the receipt of type and data information read from the // remote side of a connection. type Decoder struct { mutex sync.Mutex // each item must be received atomically r io.Reader // source of the data - buf bytes.Buffer // buffer for more efficient i/o from r + buf decBuffer // buffer for more efficient i/o from r wireType map[typeId]*wireType // map from remote ID to local description decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines ignorerCache map[typeId]**decEngine // ditto for ignored objects freeList *decoderState // list of free decoderStates; avoids reallocation countBuf []byte // used for decoding integers while parsing messages - tmp []byte // temporary storage for i/o; saves reallocating err error } @@ -75,9 +78,7 @@ func (dec *Decoder) recvMessage() bool { dec.err = err return false } - // Upper limit of 1GB, allowing room to grow a little without overflow. - // TODO: We might want more control over this limit. - if nbytes >= 1<<30 { + if nbytes >= tooBig { dec.err = errBadCount return false } @@ -87,37 +88,17 @@ func (dec *Decoder) recvMessage() bool { // readMessage reads the next nbytes bytes from the input. func (dec *Decoder) readMessage(nbytes int) { - // Allocate the dec.tmp buffer, up to 10KB. - const maxBuf = 10 * 1024 - nTmp := nbytes - if nTmp > maxBuf { - nTmp = maxBuf - } - if cap(dec.tmp) < nTmp { - nAlloc := nTmp + 100 // A little extra for growth. - if nAlloc > maxBuf { - nAlloc = maxBuf - } - dec.tmp = make([]byte, nAlloc) + if dec.buf.Len() != 0 { + // The buffer should always be empty now. + panic("non-empty decoder buffer") } - dec.tmp = dec.tmp[:nTmp] - // Read the data - dec.buf.Grow(nbytes) - for nbytes > 0 { - if nbytes < nTmp { - dec.tmp = dec.tmp[:nbytes] - } - var nRead int - nRead, dec.err = io.ReadFull(dec.r, dec.tmp) - if dec.err != nil { - if dec.err == io.EOF { - dec.err = io.ErrUnexpectedEOF - } - return + dec.buf.Size(nbytes) + _, dec.err = io.ReadFull(dec.r, dec.buf.Bytes()) + if dec.err != nil { + if dec.err == io.EOF { + dec.err = io.ErrUnexpectedEOF } - dec.buf.Write(dec.tmp) - nbytes -= nRead } } @@ -209,7 +190,7 @@ func (dec *Decoder) Decode(e interface{}) error { // Otherwise, it stores the value into v. In that case, v must represent // a non-nil pointer to data or be an assignable reflect.Value (v.CanSet()) // If the input is at EOF, DecodeValue returns io.EOF and -// does not modify e. +// does not modify v. func (dec *Decoder) DecodeValue(v reflect.Value) error { if v.IsValid() { if v.Kind() == reflect.Ptr && !v.IsNil() { diff --git a/libgo/go/encoding/gob/enc_helpers.go b/libgo/go/encoding/gob/enc_helpers.go new file mode 100644 index 0000000..804e539 --- /dev/null +++ b/libgo/go/encoding/gob/enc_helpers.go @@ -0,0 +1,414 @@ +// Created by encgen --output enc_helpers.go; DO NOT EDIT + +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gob + +import ( + "reflect" +) + +var encArrayHelper = map[reflect.Kind]encHelper{ + reflect.Bool: encBoolArray, + reflect.Complex64: encComplex64Array, + reflect.Complex128: encComplex128Array, + reflect.Float32: encFloat32Array, + reflect.Float64: encFloat64Array, + reflect.Int: encIntArray, + reflect.Int16: encInt16Array, + reflect.Int32: encInt32Array, + reflect.Int64: encInt64Array, + reflect.Int8: encInt8Array, + reflect.String: encStringArray, + reflect.Uint: encUintArray, + reflect.Uint16: encUint16Array, + reflect.Uint32: encUint32Array, + reflect.Uint64: encUint64Array, + reflect.Uintptr: encUintptrArray, +} + +var encSliceHelper = map[reflect.Kind]encHelper{ + reflect.Bool: encBoolSlice, + reflect.Complex64: encComplex64Slice, + reflect.Complex128: encComplex128Slice, + reflect.Float32: encFloat32Slice, + reflect.Float64: encFloat64Slice, + reflect.Int: encIntSlice, + reflect.Int16: encInt16Slice, + reflect.Int32: encInt32Slice, + reflect.Int64: encInt64Slice, + reflect.Int8: encInt8Slice, + reflect.String: encStringSlice, + reflect.Uint: encUintSlice, + reflect.Uint16: encUint16Slice, + reflect.Uint32: encUint32Slice, + reflect.Uint64: encUint64Slice, + reflect.Uintptr: encUintptrSlice, +} + +func encBoolArray(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encBoolSlice(state, v.Slice(0, v.Len())) +} + +func encBoolSlice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]bool) + if !ok { + // It is kind bool but not type bool. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != false || state.sendZero { + if x { + state.encodeUint(1) + } else { + state.encodeUint(0) + } + } + } + return true +} + +func encComplex64Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encComplex64Slice(state, v.Slice(0, v.Len())) +} + +func encComplex64Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]complex64) + if !ok { + // It is kind complex64 but not type complex64. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0+0i || state.sendZero { + rpart := floatBits(float64(real(x))) + ipart := floatBits(float64(imag(x))) + state.encodeUint(rpart) + state.encodeUint(ipart) + } + } + return true +} + +func encComplex128Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encComplex128Slice(state, v.Slice(0, v.Len())) +} + +func encComplex128Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]complex128) + if !ok { + // It is kind complex128 but not type complex128. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0+0i || state.sendZero { + rpart := floatBits(real(x)) + ipart := floatBits(imag(x)) + state.encodeUint(rpart) + state.encodeUint(ipart) + } + } + return true +} + +func encFloat32Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encFloat32Slice(state, v.Slice(0, v.Len())) +} + +func encFloat32Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]float32) + if !ok { + // It is kind float32 but not type float32. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + bits := floatBits(float64(x)) + state.encodeUint(bits) + } + } + return true +} + +func encFloat64Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encFloat64Slice(state, v.Slice(0, v.Len())) +} + +func encFloat64Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]float64) + if !ok { + // It is kind float64 but not type float64. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + bits := floatBits(x) + state.encodeUint(bits) + } + } + return true +} + +func encIntArray(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encIntSlice(state, v.Slice(0, v.Len())) +} + +func encIntSlice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]int) + if !ok { + // It is kind int but not type int. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeInt(int64(x)) + } + } + return true +} + +func encInt16Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encInt16Slice(state, v.Slice(0, v.Len())) +} + +func encInt16Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]int16) + if !ok { + // It is kind int16 but not type int16. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeInt(int64(x)) + } + } + return true +} + +func encInt32Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encInt32Slice(state, v.Slice(0, v.Len())) +} + +func encInt32Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]int32) + if !ok { + // It is kind int32 but not type int32. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeInt(int64(x)) + } + } + return true +} + +func encInt64Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encInt64Slice(state, v.Slice(0, v.Len())) +} + +func encInt64Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]int64) + if !ok { + // It is kind int64 but not type int64. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeInt(x) + } + } + return true +} + +func encInt8Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encInt8Slice(state, v.Slice(0, v.Len())) +} + +func encInt8Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]int8) + if !ok { + // It is kind int8 but not type int8. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeInt(int64(x)) + } + } + return true +} + +func encStringArray(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encStringSlice(state, v.Slice(0, v.Len())) +} + +func encStringSlice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]string) + if !ok { + // It is kind string but not type string. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != "" || state.sendZero { + state.encodeUint(uint64(len(x))) + state.b.WriteString(x) + } + } + return true +} + +func encUintArray(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encUintSlice(state, v.Slice(0, v.Len())) +} + +func encUintSlice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]uint) + if !ok { + // It is kind uint but not type uint. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeUint(uint64(x)) + } + } + return true +} + +func encUint16Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encUint16Slice(state, v.Slice(0, v.Len())) +} + +func encUint16Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]uint16) + if !ok { + // It is kind uint16 but not type uint16. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeUint(uint64(x)) + } + } + return true +} + +func encUint32Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encUint32Slice(state, v.Slice(0, v.Len())) +} + +func encUint32Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]uint32) + if !ok { + // It is kind uint32 but not type uint32. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeUint(uint64(x)) + } + } + return true +} + +func encUint64Array(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encUint64Slice(state, v.Slice(0, v.Len())) +} + +func encUint64Slice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]uint64) + if !ok { + // It is kind uint64 but not type uint64. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeUint(x) + } + } + return true +} + +func encUintptrArray(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return encUintptrSlice(state, v.Slice(0, v.Len())) +} + +func encUintptrSlice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]uintptr) + if !ok { + // It is kind uintptr but not type uintptr. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != 0 || state.sendZero { + state.encodeUint(uint64(x)) + } + } + return true +} diff --git a/libgo/go/encoding/gob/encgen.go b/libgo/go/encoding/gob/encgen.go new file mode 100644 index 0000000..efdd928 --- /dev/null +++ b/libgo/go/encoding/gob/encgen.go @@ -0,0 +1,218 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// encgen writes the helper functions for encoding. Intended to be +// used with go generate; see the invocation in encode.go. + +// TODO: We could do more by being unsafe. Add a -unsafe flag? + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "log" + "os" +) + +var output = flag.String("output", "enc_helpers.go", "file name to write") + +type Type struct { + lower string + upper string + zero string + encoder string +} + +var types = []Type{ + { + "bool", + "Bool", + "false", + `if x { + state.encodeUint(1) + } else { + state.encodeUint(0) + }`, + }, + { + "complex64", + "Complex64", + "0+0i", + `rpart := floatBits(float64(real(x))) + ipart := floatBits(float64(imag(x))) + state.encodeUint(rpart) + state.encodeUint(ipart)`, + }, + { + "complex128", + "Complex128", + "0+0i", + `rpart := floatBits(real(x)) + ipart := floatBits(imag(x)) + state.encodeUint(rpart) + state.encodeUint(ipart)`, + }, + { + "float32", + "Float32", + "0", + `bits := floatBits(float64(x)) + state.encodeUint(bits)`, + }, + { + "float64", + "Float64", + "0", + `bits := floatBits(x) + state.encodeUint(bits)`, + }, + { + "int", + "Int", + "0", + `state.encodeInt(int64(x))`, + }, + { + "int16", + "Int16", + "0", + `state.encodeInt(int64(x))`, + }, + { + "int32", + "Int32", + "0", + `state.encodeInt(int64(x))`, + }, + { + "int64", + "Int64", + "0", + `state.encodeInt(x)`, + }, + { + "int8", + "Int8", + "0", + `state.encodeInt(int64(x))`, + }, + { + "string", + "String", + `""`, + `state.encodeUint(uint64(len(x))) + state.b.WriteString(x)`, + }, + { + "uint", + "Uint", + "0", + `state.encodeUint(uint64(x))`, + }, + { + "uint16", + "Uint16", + "0", + `state.encodeUint(uint64(x))`, + }, + { + "uint32", + "Uint32", + "0", + `state.encodeUint(uint64(x))`, + }, + { + "uint64", + "Uint64", + "0", + `state.encodeUint(x)`, + }, + { + "uintptr", + "Uintptr", + "0", + `state.encodeUint(uint64(x))`, + }, + // uint8 Handled separately. +} + +func main() { + log.SetFlags(0) + log.SetPrefix("encgen: ") + flag.Parse() + if flag.NArg() != 0 { + log.Fatal("usage: encgen [--output filename]") + } + var b bytes.Buffer + fmt.Fprintf(&b, "// Created by encgen --output %s; DO NOT EDIT\n", *output) + fmt.Fprint(&b, header) + printMaps(&b, "Array") + fmt.Fprint(&b, "\n") + printMaps(&b, "Slice") + for _, t := range types { + fmt.Fprintf(&b, arrayHelper, t.lower, t.upper) + fmt.Fprintf(&b, sliceHelper, t.lower, t.upper, t.zero, t.encoder) + } + source, err := format.Source(b.Bytes()) + if err != nil { + log.Fatal("source format error:", err) + } + fd, err := os.Create(*output) + _, err = fd.Write(source) + if err != nil { + log.Fatal(err) + } +} + +func printMaps(b *bytes.Buffer, upperClass string) { + fmt.Fprintf(b, "var enc%sHelper = map[reflect.Kind]encHelper{\n", upperClass) + for _, t := range types { + fmt.Fprintf(b, "reflect.%s: enc%s%s,\n", t.upper, t.upper, upperClass) + } + fmt.Fprintf(b, "}\n") +} + +const header = ` +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gob + +import ( + "reflect" +) + +` + +const arrayHelper = ` +func enc%[2]sArray(state *encoderState, v reflect.Value) bool { + // Can only slice if it is addressable. + if !v.CanAddr() { + return false + } + return enc%[2]sSlice(state, v.Slice(0, v.Len())) +} +` + +const sliceHelper = ` +func enc%[2]sSlice(state *encoderState, v reflect.Value) bool { + slice, ok := v.Interface().([]%[1]s) + if !ok { + // It is kind %[1]s but not type %[1]s. TODO: We can handle this unsafely. + return false + } + for _, x := range slice { + if x != %[3]s || state.sendZero { + %[4]s + } + } + return true +} +` diff --git a/libgo/go/encoding/gob/encode.go b/libgo/go/encoding/gob/encode.go index 7831c02..f66279f 100644 --- a/libgo/go/encoding/gob/encode.go +++ b/libgo/go/encoding/gob/encode.go @@ -2,17 +2,19 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:generate go run encgen.go -output enc_helpers.go + package gob import ( - "bytes" "encoding" "math" "reflect" - "unsafe" ) -const uint64Size = int(unsafe.Sizeof(uint64(0))) +const uint64Size = 8 + +type encHelper func(state *encoderState, v reflect.Value) bool // encoderState is the global execution state of an instance of the encoder. // Field numbers are delta encoded and always increase. The field @@ -20,14 +22,46 @@ const uint64Size = int(unsafe.Sizeof(uint64(0))) // 0 terminates the structure. type encoderState struct { enc *Encoder - b *bytes.Buffer + b *encBuffer sendZero bool // encoding an array element or map key/value pair; send zero values fieldnum int // the last field number written. buf [1 + uint64Size]byte // buffer used by the encoder; here to avoid allocation. next *encoderState // for free list } -func (enc *Encoder) newEncoderState(b *bytes.Buffer) *encoderState { +// encBuffer is an extremely simple, fast implementation of a write-only byte buffer. +// It never returns a non-nil error, but Write returns an error value so it matches io.Writer. +type encBuffer struct { + data []byte + scratch [64]byte +} + +func (e *encBuffer) WriteByte(c byte) { + e.data = append(e.data, c) +} + +func (e *encBuffer) Write(p []byte) (int, error) { + e.data = append(e.data, p...) + return len(p), nil +} + +func (e *encBuffer) WriteString(s string) { + e.data = append(e.data, s...) +} + +func (e *encBuffer) Len() int { + return len(e.data) +} + +func (e *encBuffer) Bytes() []byte { + return e.data +} + +func (e *encBuffer) Reset() { + e.data = e.data[0:0] +} + +func (enc *Encoder) newEncoderState(b *encBuffer) *encoderState { e := enc.freeList if e == nil { e = new(encoderState) @@ -38,6 +72,9 @@ func (enc *Encoder) newEncoderState(b *bytes.Buffer) *encoderState { e.sendZero = false e.fieldnum = 0 e.b = b + if len(b.data) == 0 { + b.data = b.scratch[0:0] + } return e } @@ -54,10 +91,7 @@ func (enc *Encoder) freeEncoderState(e *encoderState) { // encodeUint writes an encoded unsigned integer to state.b. func (state *encoderState) encodeUint(x uint64) { if x <= 0x7F { - err := state.b.WriteByte(uint8(x)) - if err != nil { - error_(err) - } + state.b.WriteByte(uint8(x)) return } i := uint64Size @@ -67,10 +101,7 @@ func (state *encoderState) encodeUint(x uint64) { i-- } state.buf[i] = uint8(i - uint64Size) // = loop count, negated - _, err := state.b.Write(state.buf[i : uint64Size+1]) - if err != nil { - error_(err) - } + state.b.Write(state.buf[i : uint64Size+1]) } // encodeInt writes an encoded signed integer to state.w. @@ -87,14 +118,14 @@ func (state *encoderState) encodeInt(i int64) { } // encOp is the signature of an encoding operator for a given type. -type encOp func(i *encInstr, state *encoderState, p unsafe.Pointer) +type encOp func(i *encInstr, state *encoderState, v reflect.Value) // The 'instructions' of the encoding machine type encInstr struct { - op encOp - field int // field number - indir int // how many pointer indirections to reach the value in the struct - offset uintptr // offset in the structure of the field to encode + op encOp + field int // field number in input + index []int // struct index + indir int // how many pointer indirections to reach the value in the struct } // update emits a field number and updates the state to record its value for delta encoding. @@ -115,20 +146,20 @@ func (state *encoderState) update(instr *encInstr) { // encoded integer, followed by the field data in its appropriate // format. -// encIndirect dereferences p indir times and returns the result. -func encIndirect(p unsafe.Pointer, indir int) unsafe.Pointer { +// encIndirect dereferences pv indir times and returns the result. +func encIndirect(pv reflect.Value, indir int) reflect.Value { for ; indir > 0; indir-- { - p = *(*unsafe.Pointer)(p) - if p == nil { - return unsafe.Pointer(nil) + if pv.IsNil() { + break } + pv = pv.Elem() } - return p + return pv } -// encBool encodes the bool with address p as an unsigned 0 or 1. -func encBool(i *encInstr, state *encoderState, p unsafe.Pointer) { - b := *(*bool)(p) +// encBool encodes the bool referenced by v as an unsigned 0 or 1. +func encBool(i *encInstr, state *encoderState, v reflect.Value) { + b := v.Bool() if b || state.sendZero { state.update(i) if b { @@ -139,102 +170,21 @@ func encBool(i *encInstr, state *encoderState, p unsafe.Pointer) { } } -// encInt encodes the int with address p. -func encInt(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := int64(*(*int)(p)) - if v != 0 || state.sendZero { - state.update(i) - state.encodeInt(v) - } -} - -// encUint encodes the uint with address p. -func encUint(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := uint64(*(*uint)(p)) - if v != 0 || state.sendZero { +// encInt encodes the signed integer (int int8 int16 int32 int64) referenced by v. +func encInt(i *encInstr, state *encoderState, v reflect.Value) { + value := v.Int() + if value != 0 || state.sendZero { state.update(i) - state.encodeUint(v) + state.encodeInt(value) } } -// encInt8 encodes the int8 with address p. -func encInt8(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := int64(*(*int8)(p)) - if v != 0 || state.sendZero { +// encUint encodes the unsigned integer (uint uint8 uint16 uint32 uint64 uintptr) referenced by v. +func encUint(i *encInstr, state *encoderState, v reflect.Value) { + value := v.Uint() + if value != 0 || state.sendZero { state.update(i) - state.encodeInt(v) - } -} - -// encUint8 encodes the uint8 with address p. -func encUint8(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := uint64(*(*uint8)(p)) - if v != 0 || state.sendZero { - state.update(i) - state.encodeUint(v) - } -} - -// encInt16 encodes the int16 with address p. -func encInt16(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := int64(*(*int16)(p)) - if v != 0 || state.sendZero { - state.update(i) - state.encodeInt(v) - } -} - -// encUint16 encodes the uint16 with address p. -func encUint16(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := uint64(*(*uint16)(p)) - if v != 0 || state.sendZero { - state.update(i) - state.encodeUint(v) - } -} - -// encInt32 encodes the int32 with address p. -func encInt32(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := int64(*(*int32)(p)) - if v != 0 || state.sendZero { - state.update(i) - state.encodeInt(v) - } -} - -// encUint encodes the uint32 with address p. -func encUint32(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := uint64(*(*uint32)(p)) - if v != 0 || state.sendZero { - state.update(i) - state.encodeUint(v) - } -} - -// encInt64 encodes the int64 with address p. -func encInt64(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := *(*int64)(p) - if v != 0 || state.sendZero { - state.update(i) - state.encodeInt(v) - } -} - -// encInt64 encodes the uint64 with address p. -func encUint64(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := *(*uint64)(p) - if v != 0 || state.sendZero { - state.update(i) - state.encodeUint(v) - } -} - -// encUintptr encodes the uintptr with address p. -func encUintptr(i *encInstr, state *encoderState, p unsafe.Pointer) { - v := uint64(*(*uintptr)(p)) - if v != 0 || state.sendZero { - state.update(i) - state.encodeUint(v) + state.encodeUint(value) } } @@ -255,42 +205,20 @@ func floatBits(f float64) uint64 { return v } -// encFloat32 encodes the float32 with address p. -func encFloat32(i *encInstr, state *encoderState, p unsafe.Pointer) { - f := *(*float32)(p) +// encFloat encodes the floating point value (float32 float64) referenced by v. +func encFloat(i *encInstr, state *encoderState, v reflect.Value) { + f := v.Float() if f != 0 || state.sendZero { - v := floatBits(float64(f)) + bits := floatBits(f) state.update(i) - state.encodeUint(v) + state.encodeUint(bits) } } -// encFloat64 encodes the float64 with address p. -func encFloat64(i *encInstr, state *encoderState, p unsafe.Pointer) { - f := *(*float64)(p) - if f != 0 || state.sendZero { - state.update(i) - v := floatBits(f) - state.encodeUint(v) - } -} - -// encComplex64 encodes the complex64 with address p. +// encComplex encodes the complex value (complex64 complex128) referenced by v. // Complex numbers are just a pair of floating-point numbers, real part first. -func encComplex64(i *encInstr, state *encoderState, p unsafe.Pointer) { - c := *(*complex64)(p) - if c != 0+0i || state.sendZero { - rpart := floatBits(float64(real(c))) - ipart := floatBits(float64(imag(c))) - state.update(i) - state.encodeUint(rpart) - state.encodeUint(ipart) - } -} - -// encComplex128 encodes the complex128 with address p. -func encComplex128(i *encInstr, state *encoderState, p unsafe.Pointer) { - c := *(*complex128)(p) +func encComplex(i *encInstr, state *encoderState, v reflect.Value) { + c := v.Complex() if c != 0+0i || state.sendZero { rpart := floatBits(real(c)) ipart := floatBits(imag(c)) @@ -300,10 +228,10 @@ func encComplex128(i *encInstr, state *encoderState, p unsafe.Pointer) { } } -// encUint8Array encodes the byte slice whose header has address p. +// encUint8Array encodes the byte array referenced by v. // Byte arrays are encoded as an unsigned count followed by the raw bytes. -func encUint8Array(i *encInstr, state *encoderState, p unsafe.Pointer) { - b := *(*[]byte)(p) +func encUint8Array(i *encInstr, state *encoderState, v reflect.Value) { + b := v.Bytes() if len(b) > 0 || state.sendZero { state.update(i) state.encodeUint(uint64(len(b))) @@ -311,10 +239,10 @@ func encUint8Array(i *encInstr, state *encoderState, p unsafe.Pointer) { } } -// encString encodes the string whose header has address p. +// encString encodes the string referenced by v. // Strings are encoded as an unsigned count followed by the raw bytes. -func encString(i *encInstr, state *encoderState, p unsafe.Pointer) { - s := *(*string)(p) +func encString(i *encInstr, state *encoderState, v reflect.Value) { + s := v.String() if len(s) > 0 || state.sendZero { state.update(i) state.encodeUint(uint64(len(s))) @@ -324,7 +252,7 @@ func encString(i *encInstr, state *encoderState, p unsafe.Pointer) { // encStructTerminator encodes the end of an encoded struct // as delta field number of 0. -func encStructTerminator(i *encInstr, state *encoderState, p unsafe.Pointer) { +func encStructTerminator(i *encInstr, state *encoderState, v reflect.Value) { state.encodeUint(0) } @@ -338,60 +266,83 @@ type encEngine struct { const singletonField = 0 +// valid reports whether the value is valid and a non-nil pointer. +// (Slices, maps, and chans take care of themselves.) +func valid(v reflect.Value) bool { + switch v.Kind() { + case reflect.Invalid: + return false + case reflect.Ptr: + return !v.IsNil() + } + return true +} + // encodeSingle encodes a single top-level non-struct value. -func (enc *Encoder) encodeSingle(b *bytes.Buffer, engine *encEngine, basep unsafe.Pointer) { +func (enc *Encoder) encodeSingle(b *encBuffer, engine *encEngine, value reflect.Value) { state := enc.newEncoderState(b) + defer enc.freeEncoderState(state) state.fieldnum = singletonField // There is no surrounding struct to frame the transmission, so we must // generate data even if the item is zero. To do this, set sendZero. state.sendZero = true instr := &engine.instr[singletonField] - p := basep // offset will be zero if instr.indir > 0 { - if p = encIndirect(p, instr.indir); p == nil { - return - } + value = encIndirect(value, instr.indir) + } + if valid(value) { + instr.op(instr, state, value) } - instr.op(instr, state, p) - enc.freeEncoderState(state) } // encodeStruct encodes a single struct value. -func (enc *Encoder) encodeStruct(b *bytes.Buffer, engine *encEngine, basep unsafe.Pointer) { +func (enc *Encoder) encodeStruct(b *encBuffer, engine *encEngine, value reflect.Value) { + if !valid(value) { + return + } state := enc.newEncoderState(b) + defer enc.freeEncoderState(state) state.fieldnum = -1 for i := 0; i < len(engine.instr); i++ { instr := &engine.instr[i] - p := unsafe.Pointer(uintptr(basep) + instr.offset) + if i >= value.NumField() { + // encStructTerminator + instr.op(instr, state, reflect.Value{}) + break + } + field := value.FieldByIndex(instr.index) if instr.indir > 0 { - if p = encIndirect(p, instr.indir); p == nil { + field = encIndirect(field, instr.indir) + // TODO: Is field guaranteed valid? If so we could avoid this check. + if !valid(field) { continue } } - instr.op(instr, state, p) + instr.op(instr, state, field) } - enc.freeEncoderState(state) } -// encodeArray encodes the array whose 0th element is at p. -func (enc *Encoder) encodeArray(b *bytes.Buffer, p unsafe.Pointer, op encOp, elemWid uintptr, elemIndir int, length int) { +// encodeArray encodes an array. +func (enc *Encoder) encodeArray(b *encBuffer, value reflect.Value, op encOp, elemIndir int, length int, helper encHelper) { state := enc.newEncoderState(b) + defer enc.freeEncoderState(state) state.fieldnum = -1 state.sendZero = true state.encodeUint(uint64(length)) + if helper != nil && helper(state, value) { + return + } for i := 0; i < length; i++ { - elemp := p + elem := value.Index(i) if elemIndir > 0 { - up := encIndirect(elemp, elemIndir) - if up == nil { + elem = encIndirect(elem, elemIndir) + // TODO: Is elem guaranteed valid? If so we could avoid this check. + if !valid(elem) { errorf("encodeArray: nil element") } - elemp = up } - op(nil, state, elemp) - p = unsafe.Pointer(uintptr(p) + elemWid) + op(nil, state, elem) } - enc.freeEncoderState(state) } // encodeReflectValue is a helper for maps. It encodes the value v. @@ -402,13 +353,11 @@ func encodeReflectValue(state *encoderState, v reflect.Value, op encOp, indir in if !v.IsValid() { errorf("encodeReflectValue: nil element") } - op(nil, state, unsafeAddr(v)) + op(nil, state, v) } // encodeMap encodes a map as unsigned count followed by key:value pairs. -// Because map internals are not exposed, we must use reflection rather than -// addresses. -func (enc *Encoder) encodeMap(b *bytes.Buffer, mv reflect.Value, keyOp, elemOp encOp, keyIndir, elemIndir int) { +func (enc *Encoder) encodeMap(b *encBuffer, mv reflect.Value, keyOp, elemOp encOp, keyIndir, elemIndir int) { state := enc.newEncoderState(b) state.fieldnum = -1 state.sendZero = true @@ -426,7 +375,7 @@ func (enc *Encoder) encodeMap(b *bytes.Buffer, mv reflect.Value, keyOp, elemOp e // by the type identifier (which might require defining that type right now), followed // by the concrete value. A nil value gets sent as the empty string for the name, // followed by no value. -func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) { +func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) { // Gobs can encode nil interface values but not typed interface // values holding nil pointers, since nil pointers point to no value. elem := iv.Elem() @@ -450,10 +399,7 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) { } // Send the name. state.encodeUint(uint64(len(name))) - _, err := state.b.WriteString(name) - if err != nil { - error_(err) - } + state.b.WriteString(name) // Define the type id if necessary. enc.sendTypeDescriptor(enc.writer(), state, ut) // Send the type id. @@ -461,7 +407,7 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) { // Encode the value into a new buffer. Any nested type definitions // should be written to b, before the encoded value. enc.pushWriter(b) - data := new(bytes.Buffer) + data := new(encBuffer) data.Write(spaceForLength) enc.encode(data, elem, ut) if enc.err != nil { @@ -470,7 +416,7 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) { enc.popWriter() enc.writeMessage(b, data) if enc.err != nil { - error_(err) + error_(enc.err) } enc.freeEncoderState(state) } @@ -512,7 +458,7 @@ func isZero(val reflect.Value) bool { // encGobEncoder encodes a value that implements the GobEncoder interface. // The data is sent as a byte array. -func (enc *Encoder) encodeGobEncoder(b *bytes.Buffer, ut *userTypeInfo, v reflect.Value) { +func (enc *Encoder) encodeGobEncoder(b *encBuffer, ut *userTypeInfo, v reflect.Value) { // TODO: should we catch panics from the called method? var data []byte @@ -539,30 +485,30 @@ func (enc *Encoder) encodeGobEncoder(b *bytes.Buffer, ut *userTypeInfo, v reflec var encOpTable = [...]encOp{ reflect.Bool: encBool, reflect.Int: encInt, - reflect.Int8: encInt8, - reflect.Int16: encInt16, - reflect.Int32: encInt32, - reflect.Int64: encInt64, + reflect.Int8: encInt, + reflect.Int16: encInt, + reflect.Int32: encInt, + reflect.Int64: encInt, reflect.Uint: encUint, - reflect.Uint8: encUint8, - reflect.Uint16: encUint16, - reflect.Uint32: encUint32, - reflect.Uint64: encUint64, - reflect.Uintptr: encUintptr, - reflect.Float32: encFloat32, - reflect.Float64: encFloat64, - reflect.Complex64: encComplex64, - reflect.Complex128: encComplex128, + reflect.Uint8: encUint, + reflect.Uint16: encUint, + reflect.Uint32: encUint, + reflect.Uint64: encUint, + reflect.Uintptr: encUint, + reflect.Float32: encFloat, + reflect.Float64: encFloat, + reflect.Complex64: encComplex, + reflect.Complex128: encComplex, reflect.String: encString, } // encOpFor returns (a pointer to) the encoding op for the base type under rt and // the indirection count to reach it. -func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp) (*encOp, int) { +func encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp, building map[*typeInfo]bool) (*encOp, int) { ut := userType(rt) // If the type implements GobEncoder, we handle it without further processing. if ut.externalEnc != 0 { - return enc.gobEncodeOpFor(ut) + return gobEncodeOpFor(ut) } // If this type is already in progress, it's a recursive type (e.g. map[string]*T). // Return the pointer to the op we're already building. @@ -586,31 +532,27 @@ func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp break } // Slices have a header; we decode it to find the underlying array. - elemOp, elemIndir := enc.encOpFor(t.Elem(), inProgress) - op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { - slice := (*reflect.SliceHeader)(p) - if !state.sendZero && slice.Len == 0 { + elemOp, elemIndir := encOpFor(t.Elem(), inProgress, building) + helper := encSliceHelper[t.Elem().Kind()] + op = func(i *encInstr, state *encoderState, slice reflect.Value) { + if !state.sendZero && slice.Len() == 0 { return } state.update(i) - state.enc.encodeArray(state.b, unsafe.Pointer(slice.Data), *elemOp, t.Elem().Size(), elemIndir, int(slice.Len)) + state.enc.encodeArray(state.b, slice, *elemOp, elemIndir, slice.Len(), helper) } case reflect.Array: // True arrays have size in the type. - elemOp, elemIndir := enc.encOpFor(t.Elem(), inProgress) - op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { + elemOp, elemIndir := encOpFor(t.Elem(), inProgress, building) + helper := encArrayHelper[t.Elem().Kind()] + op = func(i *encInstr, state *encoderState, array reflect.Value) { state.update(i) - state.enc.encodeArray(state.b, p, *elemOp, t.Elem().Size(), elemIndir, t.Len()) + state.enc.encodeArray(state.b, array, *elemOp, elemIndir, array.Len(), helper) } case reflect.Map: - keyOp, keyIndir := enc.encOpFor(t.Key(), inProgress) - elemOp, elemIndir := enc.encOpFor(t.Elem(), inProgress) - op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { - // Maps cannot be accessed by moving addresses around the way - // that slices etc. can. We must recover a full reflection value for - // the iteration. - v := reflect.NewAt(t, unsafe.Pointer(p)).Elem() - mv := reflect.Indirect(v) + keyOp, keyIndir := encOpFor(t.Key(), inProgress, building) + elemOp, elemIndir := encOpFor(t.Elem(), inProgress, building) + op = func(i *encInstr, state *encoderState, mv reflect.Value) { // We send zero-length (but non-nil) maps because the // receiver might want to use the map. (Maps don't use append.) if !state.sendZero && mv.IsNil() { @@ -621,19 +563,16 @@ func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp } case reflect.Struct: // Generate a closure that calls out to the engine for the nested type. - enc.getEncEngine(userType(typ)) + getEncEngine(userType(typ), building) info := mustGetTypeInfo(typ) - op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { + op = func(i *encInstr, state *encoderState, sv reflect.Value) { state.update(i) // indirect through info to delay evaluation for recursive structs - state.enc.encodeStruct(state.b, info.encoder, p) + enc := info.encoder.Load().(*encEngine) + state.enc.encodeStruct(state.b, enc, sv) } case reflect.Interface: - op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { - // Interfaces transmit the name and contents of the concrete - // value they contain. - v := reflect.NewAt(t, unsafe.Pointer(p)).Elem() - iv := reflect.Indirect(v) + op = func(i *encInstr, state *encoderState, iv reflect.Value) { if !state.sendZero && (!iv.IsValid() || iv.IsNil()) { return } @@ -648,9 +587,8 @@ func (enc *Encoder) encOpFor(rt reflect.Type, inProgress map[reflect.Type]*encOp return &op, indir } -// gobEncodeOpFor returns the op for a type that is known to implement -// GobEncoder. -func (enc *Encoder) gobEncodeOpFor(ut *userTypeInfo) (*encOp, int) { +// gobEncodeOpFor returns the op for a type that is known to implement GobEncoder. +func gobEncodeOpFor(ut *userTypeInfo) (*encOp, int) { rt := ut.user if ut.encIndir == -1 { rt = reflect.PtrTo(rt) @@ -660,13 +598,13 @@ func (enc *Encoder) gobEncodeOpFor(ut *userTypeInfo) (*encOp, int) { } } var op encOp - op = func(i *encInstr, state *encoderState, p unsafe.Pointer) { - var v reflect.Value + op = func(i *encInstr, state *encoderState, v reflect.Value) { if ut.encIndir == -1 { // Need to climb up one level to turn value into pointer. - v = reflect.NewAt(rt, unsafe.Pointer(&p)).Elem() - } else { - v = reflect.NewAt(rt, p).Elem() + if !v.CanAddr() { + errorf("unaddressable value of type %s", rt) + } + v = v.Addr() } if !state.sendZero && isZero(v) { return @@ -678,7 +616,7 @@ func (enc *Encoder) gobEncodeOpFor(ut *userTypeInfo) (*encOp, int) { } // compileEnc returns the engine to compile the type. -func (enc *Encoder) compileEnc(ut *userTypeInfo) *encEngine { +func compileEnc(ut *userTypeInfo, building map[*typeInfo]bool) *encEngine { srt := ut.base engine := new(encEngine) seen := make(map[reflect.Type]*encOp) @@ -692,59 +630,57 @@ func (enc *Encoder) compileEnc(ut *userTypeInfo) *encEngine { if !isSent(&f) { continue } - op, indir := enc.encOpFor(f.Type, seen) - engine.instr = append(engine.instr, encInstr{*op, wireFieldNum, indir, uintptr(f.Offset)}) + op, indir := encOpFor(f.Type, seen, building) + engine.instr = append(engine.instr, encInstr{*op, wireFieldNum, f.Index, indir}) wireFieldNum++ } if srt.NumField() > 0 && len(engine.instr) == 0 { errorf("type %s has no exported fields", rt) } - engine.instr = append(engine.instr, encInstr{encStructTerminator, 0, 0, 0}) + engine.instr = append(engine.instr, encInstr{encStructTerminator, 0, nil, 0}) } else { engine.instr = make([]encInstr, 1) - op, indir := enc.encOpFor(rt, seen) - engine.instr[0] = encInstr{*op, singletonField, indir, 0} // offset is zero + op, indir := encOpFor(rt, seen, building) + engine.instr[0] = encInstr{*op, singletonField, nil, indir} } return engine } // getEncEngine returns the engine to compile the type. -// typeLock must be held (or we're in initialization and guaranteed single-threaded). -func (enc *Encoder) getEncEngine(ut *userTypeInfo) *encEngine { - info, err1 := getTypeInfo(ut) - if err1 != nil { - error_(err1) - } - if info.encoder == nil { - // Assign the encEngine now, so recursive types work correctly. But... - info.encoder = new(encEngine) - // ... if we fail to complete building the engine, don't cache the half-built machine. - // Doing this here means we won't cache a type that is itself OK but - // that contains a nested type that won't compile. The result is consistent - // error behavior when Encode is called multiple times on the top-level type. - ok := false - defer func() { - if !ok { - info.encoder = nil - } - }() - info.encoder = enc.compileEnc(ut) - ok = true +func getEncEngine(ut *userTypeInfo, building map[*typeInfo]bool) *encEngine { + info, err := getTypeInfo(ut) + if err != nil { + error_(err) } - return info.encoder + enc, ok := info.encoder.Load().(*encEngine) + if !ok { + enc = buildEncEngine(info, ut, building) + } + return enc } -// lockAndGetEncEngine is a function that locks and compiles. -// This lets us hold the lock only while compiling, not when encoding. -func (enc *Encoder) lockAndGetEncEngine(ut *userTypeInfo) *encEngine { - typeLock.Lock() - defer typeLock.Unlock() - return enc.getEncEngine(ut) +func buildEncEngine(info *typeInfo, ut *userTypeInfo, building map[*typeInfo]bool) *encEngine { + // Check for recursive types. + if building != nil && building[info] { + return nil + } + info.encInit.Lock() + defer info.encInit.Unlock() + enc, ok := info.encoder.Load().(*encEngine) + if !ok { + if building == nil { + building = make(map[*typeInfo]bool) + } + building[info] = true + enc = compileEnc(ut, building) + info.encoder.Store(enc) + } + return enc } -func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value, ut *userTypeInfo) { +func (enc *Encoder) encode(b *encBuffer, value reflect.Value, ut *userTypeInfo) { defer catchError(&enc.err) - engine := enc.lockAndGetEncEngine(ut) + engine := getEncEngine(ut, nil) indir := ut.indir if ut.externalEnc != 0 { indir = int(ut.encIndir) @@ -753,8 +689,8 @@ func (enc *Encoder) encode(b *bytes.Buffer, value reflect.Value, ut *userTypeInf value = reflect.Indirect(value) } if ut.externalEnc == 0 && value.Type().Kind() == reflect.Struct { - enc.encodeStruct(b, engine, unsafeAddr(value)) + enc.encodeStruct(b, engine, value) } else { - enc.encodeSingle(b, engine, unsafeAddr(value)) + enc.encodeSingle(b, engine, value) } } diff --git a/libgo/go/encoding/gob/encoder.go b/libgo/go/encoding/gob/encoder.go index a3301c3..a340e47 100644 --- a/libgo/go/encoding/gob/encoder.go +++ b/libgo/go/encoding/gob/encoder.go @@ -5,7 +5,6 @@ package gob import ( - "bytes" "io" "reflect" "sync" @@ -19,7 +18,7 @@ type Encoder struct { sent map[reflect.Type]typeId // which types we've already sent countState *encoderState // stage for writing counts freeList *encoderState // list of free encoderStates; avoids reallocation - byteBuf bytes.Buffer // buffer for top-level encoderState + byteBuf encBuffer // buffer for top-level encoderState err error } @@ -34,7 +33,7 @@ func NewEncoder(w io.Writer) *Encoder { enc := new(Encoder) enc.w = []io.Writer{w} enc.sent = make(map[reflect.Type]typeId) - enc.countState = enc.newEncoderState(new(bytes.Buffer)) + enc.countState = enc.newEncoderState(new(encBuffer)) return enc } @@ -60,7 +59,7 @@ func (enc *Encoder) setError(err error) { } // writeMessage sends the data item preceded by a unsigned count of its length. -func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) { +func (enc *Encoder) writeMessage(w io.Writer, b *encBuffer) { // Space has been reserved for the length at the head of the message. // This is a little dirty: we grab the slice from the bytes.Buffer and massage // it by hand. @@ -88,9 +87,7 @@ func (enc *Encoder) sendActualType(w io.Writer, state *encoderState, ut *userTyp if _, alreadySent := enc.sent[actual]; alreadySent { return false } - typeLock.Lock() info, err := getTypeInfo(ut) - typeLock.Unlock() if err != nil { enc.setError(err) return @@ -191,9 +188,7 @@ func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *use // a singleton basic type (int, []byte etc.) at top level. We don't // need to send the type info but we do need to update enc.sent. if !sent { - typeLock.Lock() info, err := getTypeInfo(ut) - typeLock.Unlock() if err != nil { enc.setError(err) return diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index 6445ce1..0ea4c0e 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -13,6 +13,52 @@ import ( "testing" ) +// Test basic operations in a safe manner. +func TestBasicEncoderDecoder(t *testing.T) { + var values = []interface{}{ + true, + int(123), + int8(123), + int16(-12345), + int32(123456), + int64(-1234567), + uint(123), + uint8(123), + uint16(12345), + uint32(123456), + uint64(1234567), + uintptr(12345678), + float32(1.2345), + float64(1.2345678), + complex64(1.2345 + 2.3456i), + complex128(1.2345678 + 2.3456789i), + []byte("hello"), + string("hello"), + } + for _, value := range values { + b := new(bytes.Buffer) + enc := NewEncoder(b) + err := enc.Encode(value) + if err != nil { + t.Error("encoder fail:", err) + } + dec := NewDecoder(b) + result := reflect.New(reflect.TypeOf(value)) + err = dec.Decode(result.Interface()) + if err != nil { + t.Fatalf("error decoding %T: %v:", reflect.TypeOf(value), err) + } + if !reflect.DeepEqual(value, result.Elem().Interface()) { + t.Fatalf("%T: expected %v got %v", value, value, result.Elem().Interface()) + } + } +} + +type ET0 struct { + A int + B string +} + type ET2 struct { X string } @@ -40,14 +86,40 @@ type ET4 struct { func TestEncoderDecoder(t *testing.T) { b := new(bytes.Buffer) enc := NewEncoder(b) + et0 := new(ET0) + et0.A = 7 + et0.B = "gobs of fun" + err := enc.Encode(et0) + if err != nil { + t.Error("encoder fail:", err) + } + //fmt.Printf("% x %q\n", b, b) + //Debug(b) + dec := NewDecoder(b) + newEt0 := new(ET0) + err = dec.Decode(newEt0) + if err != nil { + t.Fatal("error decoding ET0:", err) + } + + if !reflect.DeepEqual(et0, newEt0) { + t.Fatalf("invalid data for et0: expected %+v; got %+v", *et0, *newEt0) + } + if b.Len() != 0 { + t.Error("not at eof;", b.Len(), "bytes left") + } + // t.FailNow() + + b = new(bytes.Buffer) + enc = NewEncoder(b) et1 := new(ET1) et1.A = 7 et1.Et2 = new(ET2) - err := enc.Encode(et1) + err = enc.Encode(et1) if err != nil { t.Error("encoder fail:", err) } - dec := NewDecoder(b) + dec = NewDecoder(b) newEt1 := new(ET1) err = dec.Decode(newEt1) if err != nil { @@ -860,3 +932,25 @@ func Test29ElementSlice(t *testing.T) { return } } + +// Don't crash, just give error when allocating a huge slice. +// Issue 8084. +func TestErrorForHugeSlice(t *testing.T) { + // Encode an int slice. + buf := new(bytes.Buffer) + slice := []int{1, 1, 1, 1, 1, 1, 1, 1, 1, 1} + err := NewEncoder(buf).Encode(slice) + if err != nil { + t.Fatal("encode:", err) + } + // Reach into the buffer and smash the count to make the encoded slice very long. + buf.Bytes()[buf.Len()-len(slice)-1] = 0xfa + // Decode and see error. + err = NewDecoder(buf).Decode(&slice) + if err == nil { + t.Fatal("decode: no error") + } + if !strings.Contains(err.Error(), "slice too big") { + t.Fatal("decode: expected slice too big error, got %s", err.Error()) + } +} diff --git a/libgo/go/encoding/gob/gobencdec_test.go b/libgo/go/encoding/gob/gobencdec_test.go index 157b772..eb76b48 100644 --- a/libgo/go/encoding/gob/gobencdec_test.go +++ b/libgo/go/encoding/gob/gobencdec_test.go @@ -279,7 +279,7 @@ func TestGobEncoderValueField(t *testing.T) { b := new(bytes.Buffer) // First a field that's a structure. enc := NewEncoder(b) - err := enc.Encode(GobTestValueEncDec{17, StringStruct{"HIJKL"}}) + err := enc.Encode(&GobTestValueEncDec{17, StringStruct{"HIJKL"}}) if err != nil { t.Fatal("encode error:", err) } @@ -326,7 +326,7 @@ func TestGobEncoderArrayField(t *testing.T) { for i := range a.A.a { a.A.a[i] = byte(i) } - err := enc.Encode(a) + err := enc.Encode(&a) if err != nil { t.Fatal("encode error:", err) } @@ -589,7 +589,8 @@ func TestGobEncoderStructSingleton(t *testing.T) { func TestGobEncoderNonStructSingleton(t *testing.T) { b := new(bytes.Buffer) enc := NewEncoder(b) - err := enc.Encode(Gobber(1234)) + var g Gobber = 1234 + err := enc.Encode(&g) if err != nil { t.Fatal("encode error:", err) } diff --git a/libgo/go/encoding/gob/timing_test.go b/libgo/go/encoding/gob/timing_test.go index 9fbb0ac..940e5ad 100644 --- a/libgo/go/encoding/gob/timing_test.go +++ b/libgo/go/encoding/gob/timing_test.go @@ -19,33 +19,57 @@ type Bench struct { D []byte } -func benchmarkEndToEnd(r io.Reader, w io.Writer, b *testing.B) { - b.StopTimer() - enc := NewEncoder(w) - dec := NewDecoder(r) - bench := &Bench{7, 3.2, "now is the time", []byte("for all good men")} - b.StartTimer() - for i := 0; i < b.N; i++ { - if enc.Encode(bench) != nil { - panic("encode error") +func benchmarkEndToEnd(b *testing.B, ctor func() interface{}, pipe func() (r io.Reader, w io.Writer, err error)) { + b.RunParallel(func(pb *testing.PB) { + r, w, err := pipe() + if err != nil { + b.Fatal("can't get pipe:", err) } - if dec.Decode(bench) != nil { - panic("decode error") + v := ctor() + enc := NewEncoder(w) + dec := NewDecoder(r) + for pb.Next() { + if err := enc.Encode(v); err != nil { + b.Fatal("encode error:", err) + } + if err := dec.Decode(v); err != nil { + b.Fatal("decode error:", err) + } } - } + }) } func BenchmarkEndToEndPipe(b *testing.B) { - r, w, err := os.Pipe() - if err != nil { - b.Fatal("can't get pipe:", err) - } - benchmarkEndToEnd(r, w, b) + benchmarkEndToEnd(b, func() interface{} { + return &Bench{7, 3.2, "now is the time", bytes.Repeat([]byte("for all good men"), 100)} + }, func() (r io.Reader, w io.Writer, err error) { + r, w, err = os.Pipe() + return + }) } func BenchmarkEndToEndByteBuffer(b *testing.B) { - var buf bytes.Buffer - benchmarkEndToEnd(&buf, &buf, b) + benchmarkEndToEnd(b, func() interface{} { + return &Bench{7, 3.2, "now is the time", bytes.Repeat([]byte("for all good men"), 100)} + }, func() (r io.Reader, w io.Writer, err error) { + var buf bytes.Buffer + return &buf, &buf, nil + }) +} + +func BenchmarkEndToEndSliceByteBuffer(b *testing.B) { + benchmarkEndToEnd(b, func() interface{} { + v := &Bench{7, 3.2, "now is the time", nil} + Register(v) + arr := make([]interface{}, 100) + for i := range arr { + arr[i] = v + } + return &arr + }, func() (r io.Reader, w io.Writer, err error) { + var buf bytes.Buffer + return &buf, &buf, nil + }) } func TestCountEncodeMallocs(t *testing.T) { @@ -103,7 +127,199 @@ func TestCountDecodeMallocs(t *testing.T) { t.Fatal("decode:", err) } }) - if allocs != 3 { - t.Fatalf("mallocs per decode of type Bench: %v; wanted 3\n", allocs) + if allocs != 4 { + t.Fatalf("mallocs per decode of type Bench: %v; wanted 4\n", allocs) + } +} + +func BenchmarkEncodeComplex128Slice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]complex128, 1000) + for i := range a { + a[i] = 1.2 + 3.4i + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf.Reset() + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeFloat64Slice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]float64, 1000) + for i := range a { + a[i] = 1.23e4 + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf.Reset() + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeInt32Slice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]int32, 1000) + for i := range a { + a[i] = 1234 + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf.Reset() + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeStringSlice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]string, 1000) + for i := range a { + a[i] = "now is the time" + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf.Reset() + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + } +} + +// benchmarkBuf is a read buffer we can reset +type benchmarkBuf struct { + offset int + data []byte +} + +func (b *benchmarkBuf) Read(p []byte) (n int, err error) { + n = copy(p, b.data[b.offset:]) + if n == 0 { + return 0, io.EOF + } + b.offset += n + return +} + +func (b *benchmarkBuf) ReadByte() (c byte, err error) { + if b.offset >= len(b.data) { + return 0, io.EOF + } + c = b.data[b.offset] + b.offset++ + return +} + +func (b *benchmarkBuf) reset() { + b.offset = 0 +} + +func BenchmarkDecodeComplex128Slice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]complex128, 1000) + for i := range a { + a[i] = 1.2 + 3.4i + } + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + x := make([]complex128, 1000) + bbuf := benchmarkBuf{data: buf.Bytes()} + b.ResetTimer() + for i := 0; i < b.N; i++ { + bbuf.reset() + dec := NewDecoder(&bbuf) + err := dec.Decode(&x) + if err != nil { + b.Fatal(i, err) + } + } +} + +func BenchmarkDecodeFloat64Slice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]float64, 1000) + for i := range a { + a[i] = 1.23e4 + } + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + x := make([]float64, 1000) + bbuf := benchmarkBuf{data: buf.Bytes()} + b.ResetTimer() + for i := 0; i < b.N; i++ { + bbuf.reset() + dec := NewDecoder(&bbuf) + err := dec.Decode(&x) + if err != nil { + b.Fatal(i, err) + } + } +} + +func BenchmarkDecodeInt32Slice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]int32, 1000) + for i := range a { + a[i] = 1234 + } + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + x := make([]int32, 1000) + bbuf := benchmarkBuf{data: buf.Bytes()} + b.ResetTimer() + for i := 0; i < b.N; i++ { + bbuf.reset() + dec := NewDecoder(&bbuf) + err := dec.Decode(&x) + if err != nil { + b.Fatal(i, err) + } + } +} + +func BenchmarkDecodeStringSlice(b *testing.B) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + a := make([]string, 1000) + for i := range a { + a[i] = "now is the time" + } + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + x := make([]string, 1000) + bbuf := benchmarkBuf{data: buf.Bytes()} + b.ResetTimer() + for i := 0; i < b.N; i++ { + bbuf.reset() + dec := NewDecoder(&bbuf) + err := dec.Decode(&x) + if err != nil { + b.Fatal(i, err) + } } } diff --git a/libgo/go/encoding/gob/type.go b/libgo/go/encoding/gob/type.go index cad1452..a49b71a 100644 --- a/libgo/go/encoding/gob/type.go +++ b/libgo/go/encoding/gob/type.go @@ -11,6 +11,7 @@ import ( "os" "reflect" "sync" + "sync/atomic" "unicode" "unicode/utf8" ) @@ -681,29 +682,51 @@ func (w *wireType) string() string { type typeInfo struct { id typeId - encoder *encEngine + encInit sync.Mutex // protects creation of encoder + encoder atomic.Value // *encEngine wire *wireType } -var typeInfoMap = make(map[reflect.Type]*typeInfo) // protected by typeLock +// typeInfoMap is an atomic pointer to map[reflect.Type]*typeInfo. +// It's updated copy-on-write. Readers just do an atomic load +// to get the current version of the map. Writers make a full copy of +// the map and atomically update the pointer to point to the new map. +// Under heavy read contention, this is significantly faster than a map +// protected by a mutex. +var typeInfoMap atomic.Value + +func lookupTypeInfo(rt reflect.Type) *typeInfo { + m, _ := typeInfoMap.Load().(map[reflect.Type]*typeInfo) + return m[rt] +} -// typeLock must be held. func getTypeInfo(ut *userTypeInfo) (*typeInfo, error) { rt := ut.base if ut.externalEnc != 0 { // We want the user type, not the base type. rt = ut.user } - info, ok := typeInfoMap[rt] - if ok { + if info := lookupTypeInfo(rt); info != nil { return info, nil } - info = new(typeInfo) + return buildTypeInfo(ut, rt) +} + +// buildTypeInfo constructs the type information for the type +// and stores it in the type info map. +func buildTypeInfo(ut *userTypeInfo, rt reflect.Type) (*typeInfo, error) { + typeLock.Lock() + defer typeLock.Unlock() + + if info := lookupTypeInfo(rt); info != nil { + return info, nil + } + gt, err := getBaseType(rt.Name(), rt) if err != nil { return nil, err } - info.id = gt.id() + info := &typeInfo{id: gt.id()} if ut.externalEnc != 0 { userType, err := getType(rt.Name(), ut, rt) @@ -719,25 +742,32 @@ func getTypeInfo(ut *userTypeInfo) (*typeInfo, error) { case xText: info.wire = &wireType{TextMarshalerT: gt} } - typeInfoMap[ut.user] = info - return info, nil + rt = ut.user + } else { + t := info.id.gobType() + switch typ := rt; typ.Kind() { + case reflect.Array: + info.wire = &wireType{ArrayT: t.(*arrayType)} + case reflect.Map: + info.wire = &wireType{MapT: t.(*mapType)} + case reflect.Slice: + // []byte == []uint8 is a special case handled separately + if typ.Elem().Kind() != reflect.Uint8 { + info.wire = &wireType{SliceT: t.(*sliceType)} + } + case reflect.Struct: + info.wire = &wireType{StructT: t.(*structType)} + } } - t := info.id.gobType() - switch typ := rt; typ.Kind() { - case reflect.Array: - info.wire = &wireType{ArrayT: t.(*arrayType)} - case reflect.Map: - info.wire = &wireType{MapT: t.(*mapType)} - case reflect.Slice: - // []byte == []uint8 is a special case handled separately - if typ.Elem().Kind() != reflect.Uint8 { - info.wire = &wireType{SliceT: t.(*sliceType)} - } - case reflect.Struct: - info.wire = &wireType{StructT: t.(*structType)} + // Create new map with old contents plus new entry. + newm := make(map[reflect.Type]*typeInfo) + m, _ := typeInfoMap.Load().(map[reflect.Type]*typeInfo) + for k, v := range m { + newm[k] = v } - typeInfoMap[rt] = info + newm[rt] = info + typeInfoMap.Store(newm) return info, nil } diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index af1c908..705bc2e 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -173,7 +173,6 @@ type decodeState struct { scan scanner nextscan scanner // for calls to nextValue savedError error - tempstr string // scratch space to avoid some allocations useNumber bool } @@ -293,6 +292,32 @@ func (d *decodeState) value(v reflect.Value) { } } +type unquotedValue struct{} + +// valueQuoted is like value but decodes a +// quoted string literal or literal null into an interface value. +// If it finds anything other than a quoted string literal or null, +// valueQuoted returns unquotedValue{}. +func (d *decodeState) valueQuoted() interface{} { + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + + case scanBeginArray: + d.array(reflect.Value{}) + + case scanBeginObject: + d.object(reflect.Value{}) + + case scanBeginLiteral: + switch v := d.literalInterface().(type) { + case nil, string: + return v + } + } + return unquotedValue{} +} + // indirect walks down v allocating pointers as needed, // until it gets to a non-pointer. // if it encounters an Unmarshaler, indirect stops and returns that. @@ -444,8 +469,10 @@ func (d *decodeState) array(v reflect.Value) { } } +var nullLiteral = []byte("null") + // object consumes an object from d.data[d.off-1:], decoding into the value v. -// the first byte of the object ('{') has been read already. +// the first byte ('{') of the object has been read already. func (d *decodeState) object(v reflect.Value) { // Check for unmarshaler. u, ut, pv := d.indirect(v, false) @@ -478,7 +505,9 @@ func (d *decodeState) object(v reflect.Value) { t := v.Type() if t.Key().Kind() != reflect.String { d.saveError(&UnmarshalTypeError{"object", v.Type()}) - break + d.off-- + d.next() // skip over { } in input + return } if v.IsNil() { v.Set(reflect.MakeMap(t)) @@ -564,9 +593,14 @@ func (d *decodeState) object(v reflect.Value) { // Read value. if destring { - d.value(reflect.ValueOf(&d.tempstr)) - d.literalStore([]byte(d.tempstr), subv, true) - d.tempstr = "" // Zero scratch space for successive values. + switch qv := d.valueQuoted().(type) { + case nil: + d.literalStore(nullLiteral, subv, false) + case string: + d.literalStore([]byte(qv), subv, true) + default: + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", item, v.Type())) + } } else { d.value(subv) } diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index 238a87f..7235969 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -406,6 +406,13 @@ var unmarshalTests = []unmarshalTest{ ptr: new(string), out: "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld", }, + + // issue 8305 + { + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: &map[time.Time]string{}, + err: &UnmarshalTypeError{"object", reflect.TypeOf(map[time.Time]string{})}, + }, } func TestMarshal(t *testing.T) { @@ -514,6 +521,7 @@ func TestUnmarshal(t *testing.T) { if tt.ptr == nil { continue } + // v = new(right-type) v := reflect.New(reflect.TypeOf(tt.ptr).Elem()) dec := NewDecoder(bytes.NewReader(in)) @@ -521,7 +529,9 @@ func TestUnmarshal(t *testing.T) { dec.UseNumber() } if err := dec.Decode(v.Interface()); !reflect.DeepEqual(err, tt.err) { - t.Errorf("#%d: %v want %v", i, err, tt.err) + t.Errorf("#%d: %v, want %v", i, err, tt.err) + continue + } else if err != nil { continue } if !reflect.DeepEqual(v.Elem().Interface(), tt.out) { @@ -1060,18 +1070,25 @@ func TestEmptyString(t *testing.T) { } } -// Test that the returned error is non-nil when trying to unmarshal null string into int, for successive ,string option -// Issue 7046 +// Test that a null for ,string is not replaced with the previous quoted string (issue 7046). +// It should also not be an error (issue 2540, issue 8587). func TestNullString(t *testing.T) { type T struct { - A int `json:",string"` - B int `json:",string"` + A int `json:",string"` + B int `json:",string"` + C *int `json:",string"` } - data := []byte(`{"A": "1", "B": null}`) + data := []byte(`{"A": "1", "B": null, "C": null}`) var s T + s.B = 1 + s.C = new(int) + *s.C = 2 err := Unmarshal(data, &s) - if err == nil { - t.Fatalf("expected error; got %v", s) + if err != nil { + t.Fatalf("Unmarshal: %v") + } + if s.B != 1 || s.C != nil { + t.Fatalf("after Unmarshal, s.B=%d, s.C=%p, want 1, nil", s.B, s.C) } } diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index 741ddd8..fca2a09 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -40,8 +40,8 @@ import ( // // Floating point, integer, and Number values encode as JSON numbers. // -// String values encode as JSON strings. InvalidUTF8Error will be returned -// if an invalid UTF-8 sequence is encountered. +// String values encode as JSON strings coerced to valid UTF-8, +// replacing invalid bytes with the Unicode replacement rune. // The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" // to keep some browsers from misinterpreting JSON output as HTML. // Ampersand "&" is also escaped to "\u0026" for the same reason. @@ -93,6 +93,8 @@ import ( // as described in the next paragraph. // An anonymous struct field with a name given in its JSON tag is treated as // having that name, rather than being anonymous. +// An anonymous struct field of interface type is treated the same as having +// that type as its name, rather than being anonymous. // // The Go visibility rules for struct fields are amended for JSON when // deciding which field to marshal or unmarshal. If there are @@ -696,12 +698,12 @@ type ptrEncoder struct { elemEnc encoderFunc } -func (pe *ptrEncoder) encode(e *encodeState, v reflect.Value, _ bool) { +func (pe *ptrEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { if v.IsNil() { e.WriteString("null") return } - pe.elemEnc(e, v.Elem(), false) + pe.elemEnc(e, v.Elem(), quoted) } func newPtrEncoder(t reflect.Type) encoderFunc { @@ -803,6 +805,9 @@ func (e *encodeState) string(s string) (int, error) { case '\r': e.WriteByte('\\') e.WriteByte('r') + case '\t': + e.WriteByte('\\') + e.WriteByte('t') default: // This encodes bytes < 0x20 except for \n and \r, // as well as <, > and &. The latter are escaped because they @@ -876,9 +881,12 @@ func (e *encodeState) stringBytes(s []byte) (int, error) { case '\r': e.WriteByte('\\') e.WriteByte('r') + case '\t': + e.WriteByte('\\') + e.WriteByte('t') default: // This encodes bytes < 0x20 except for \n and \r, - // as well as < and >. The latter are escaped because they + // as well as <, >, and &. The latter are escaped because they // can lead to security holes when user-controlled strings // are rendered into JSON and served to some browsers. e.WriteString(`\u00`) diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index 2e89a78..7abfa85 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -452,3 +452,81 @@ func TestHTMLEscape(t *testing.T) { t.Errorf("HTMLEscape(&b, []byte(m)) = %s; want %s", b.Bytes(), want.Bytes()) } } + +// golang.org/issue/8582 +func TestEncodePointerString(t *testing.T) { + type stringPointer struct { + N *int64 `json:"n,string"` + } + var n int64 = 42 + b, err := Marshal(stringPointer{N: &n}) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if got, want := string(b), `{"n":"42"}`; got != want { + t.Errorf("Marshal = %s, want %s", got, want) + } + var back stringPointer + err = Unmarshal(b, &back) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if back.N == nil { + t.Fatalf("Unmarshalled nil N field") + } + if *back.N != 42 { + t.Fatalf("*N = %d; want 42", *back.N) + } +} + +var encodeStringTests = []struct { + in string + out string +}{ + {"\x00", `"\u0000"`}, + {"\x01", `"\u0001"`}, + {"\x02", `"\u0002"`}, + {"\x03", `"\u0003"`}, + {"\x04", `"\u0004"`}, + {"\x05", `"\u0005"`}, + {"\x06", `"\u0006"`}, + {"\x07", `"\u0007"`}, + {"\x08", `"\u0008"`}, + {"\x09", `"\t"`}, + {"\x0a", `"\n"`}, + {"\x0b", `"\u000b"`}, + {"\x0c", `"\u000c"`}, + {"\x0d", `"\r"`}, + {"\x0e", `"\u000e"`}, + {"\x0f", `"\u000f"`}, + {"\x10", `"\u0010"`}, + {"\x11", `"\u0011"`}, + {"\x12", `"\u0012"`}, + {"\x13", `"\u0013"`}, + {"\x14", `"\u0014"`}, + {"\x15", `"\u0015"`}, + {"\x16", `"\u0016"`}, + {"\x17", `"\u0017"`}, + {"\x18", `"\u0018"`}, + {"\x19", `"\u0019"`}, + {"\x1a", `"\u001a"`}, + {"\x1b", `"\u001b"`}, + {"\x1c", `"\u001c"`}, + {"\x1d", `"\u001d"`}, + {"\x1e", `"\u001e"`}, + {"\x1f", `"\u001f"`}, +} + +func TestEncodeString(t *testing.T) { + for _, tt := range encodeStringTests { + b, err := Marshal(tt.in) + if err != nil { + t.Errorf("Marshal(%q): %v", tt.in, err) + continue + } + out := string(b) + if out != tt.out { + t.Errorf("Marshal(%q) = %#q, want %#q", tt.in, out, tt.out) + } + } +} diff --git a/libgo/go/encoding/json/stream.go b/libgo/go/encoding/json/stream.go index 1cb289f..9566eca 100644 --- a/libgo/go/encoding/json/stream.go +++ b/libgo/go/encoding/json/stream.go @@ -139,7 +139,6 @@ func nonSpace(b []byte) bool { // An Encoder writes JSON objects to an output stream. type Encoder struct { w io.Writer - e encodeState err error } diff --git a/libgo/go/encoding/xml/xml.go b/libgo/go/encoding/xml/xml.go index b473cb8..8c15b98 100644 --- a/libgo/go/encoding/xml/xml.go +++ b/libgo/go/encoding/xml/xml.go @@ -196,6 +196,7 @@ type Decoder struct { ns map[string]string err error line int + offset int64 unmarshalDepth int } @@ -859,9 +860,17 @@ func (d *Decoder) getc() (b byte, ok bool) { if b == '\n' { d.line++ } + d.offset++ return b, true } +// InputOffset returns the input stream byte offset of the current decoder position. +// The offset gives the location of the end of the most recently returned token +// and the beginning of the next token. +func (d *Decoder) InputOffset() int64 { + return d.offset +} + // Return saved offset. // If we did ungetc (nextByte >= 0), have to back up one. func (d *Decoder) savedOffset() int { @@ -891,6 +900,7 @@ func (d *Decoder) ungetc(b byte) { d.line-- } d.nextByte = int(b) + d.offset-- } var entity = map[string]int{ diff --git a/libgo/go/encoding/xml/xml_test.go b/libgo/go/encoding/xml/xml_test.go index 7723ab1..be995c0 100644 --- a/libgo/go/encoding/xml/xml_test.go +++ b/libgo/go/encoding/xml/xml_test.go @@ -170,7 +170,7 @@ var xmlInput = []string{ func TestRawToken(t *testing.T) { d := NewDecoder(strings.NewReader(testInput)) d.Entity = testEntity - testRawToken(t, d, rawTokens) + testRawToken(t, d, testInput, rawTokens) } const nonStrictInput = ` @@ -225,7 +225,7 @@ var nonStrictTokens = []Token{ func TestNonStrictRawToken(t *testing.T) { d := NewDecoder(strings.NewReader(nonStrictInput)) d.Strict = false - testRawToken(t, d, nonStrictTokens) + testRawToken(t, d, nonStrictInput, nonStrictTokens) } type downCaser struct { @@ -254,7 +254,7 @@ func TestRawTokenAltEncoding(t *testing.T) { } return &downCaser{t, input.(io.ByteReader)}, nil } - testRawToken(t, d, rawTokensAltEncoding) + testRawToken(t, d, testInputAltEncoding, rawTokensAltEncoding) } func TestRawTokenAltEncodingNoConverter(t *testing.T) { @@ -280,9 +280,12 @@ func TestRawTokenAltEncodingNoConverter(t *testing.T) { } } -func testRawToken(t *testing.T, d *Decoder, rawTokens []Token) { +func testRawToken(t *testing.T, d *Decoder, raw string, rawTokens []Token) { + lastEnd := int64(0) for i, want := range rawTokens { + start := d.InputOffset() have, err := d.RawToken() + end := d.InputOffset() if err != nil { t.Fatalf("token %d: unexpected error: %s", i, err) } @@ -300,6 +303,26 @@ func testRawToken(t *testing.T, d *Decoder, rawTokens []Token) { } t.Errorf("token %d = %s, want %s", i, shave, swant) } + + // Check that InputOffset returned actual token. + switch { + case start < lastEnd: + t.Errorf("token %d: position [%d,%d) for %T is before previous token", i, start, end, have) + case start >= end: + // Special case: EndElement can be synthesized. + if start == end && end == lastEnd { + break + } + t.Errorf("token %d: position [%d,%d) for %T is empty", i, start, end, have) + case end > int64(len(raw)): + t.Errorf("token %d: position [%d,%d) for %T extends beyond input", i, start, end, have) + default: + text := raw[start:end] + if strings.ContainsAny(text, "<>") && (!strings.HasPrefix(text, "<") || !strings.HasSuffix(text, ">")) { + t.Errorf("token %d: misaligned raw token %#q for %T", i, text, have) + } + } + lastEnd = end } } |