Merge branch 'autopar_rebase2' into autopar_develdevel/autopar_devel

Quickly commit changes in the rebase branch.

22 files changed, 515 insertions, 129 deletions

diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go
index fd4dd68..d809dde 100644
--- a/libgo/go/encoding/asn1/asn1.go
+++ b/libgo/go/encoding/asn1/asn1.go
@@ -313,6 +313,12 @@ func parseBase128Int(bytes []byte, initOffset int) (ret, offset int, err error)
 		}
 		ret64 <<= 7
 		b := bytes[offset]
+		// integers should be minimally encoded, so the leading octet should
+		// never be 0x80
+		if shifted == 0 && b == 0x80 {
+			err = SyntaxError{"integer is not minimally encoded"}
+			return
+		}
 		ret64 |= int64(b & 0x7f)
 		offset++
 		if b&0x80 == 0 {
@@ -1031,6 +1037,12 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) {
 // Because Unmarshal uses the reflect package, the structs
 // being written to must use upper case field names.
 //
+// After parsing b, any bytes that were leftover and not used to fill
+// val will be returned in rest. When parsing a SEQUENCE into a struct,
+// any trailing elements of the SEQUENCE that do not have matching
+// fields in val will not be included in rest, as these are considered
+// valid elements of the SEQUENCE and not trailing data.
+//
 // An ASN.1 INTEGER can be written to an int, int32, int64,
 // or *big.Int (from the math/big package).
 // If the encoded value does not fit in the Go type,
diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go
index d5649bf..8daae97 100644
--- a/libgo/go/encoding/asn1/asn1_test.go
+++ b/libgo/go/encoding/asn1/asn1_test.go
@@ -1129,3 +1129,15 @@ func TestBMPString(t *testing.T) {
 		}
 	}
 }
+
+func TestNonMinimalEncodedOID(t *testing.T) {
+	h, err := hex.DecodeString("060a2a80864886f70d01010b")
+	if err != nil {
+		t.Fatalf("failed to decode from hex string: %s", err)
+	}
+	var oid ObjectIdentifier
+	_, err = Unmarshal(h, &oid)
+	if err == nil {
+		t.Fatalf("accepted non-minimally encoded oid")
+	}
+}
diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go
index c9ae2ca..0d34d5a 100644
--- a/libgo/go/encoding/asn1/marshal.go
+++ b/libgo/go/encoding/asn1/marshal.go
@@ -5,10 +5,12 @@
 package asn1
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"math/big"
 	"reflect"
+	"sort"
 	"time"
 	"unicode/utf8"
 )
@@ -78,6 +80,48 @@ func (m multiEncoder) Encode(dst []byte) {
 	}
 }
 
+type setEncoder []encoder
+
+func (s setEncoder) Len() int {
+	var size int
+	for _, e := range s {
+		size += e.Len()
+	}
+	return size
+}
+
+func (s setEncoder) Encode(dst []byte) {
+	// Per X690 Section 11.6: The encodings of the component values of a
+	// set-of value shall appear in ascending order, the encodings being
+	// compared as octet strings with the shorter components being padded
+	// at their trailing end with 0-octets.
+	//
+	// First we encode each element to its TLV encoding and then use
+	// octetSort to get the ordering expected by X690 DER rules before
+	// writing the sorted encodings out to dst.
+	l := make([][]byte, len(s))
+	for i, e := range s {
+		l[i] = make([]byte, e.Len())
+		e.Encode(l[i])
+	}
+
+	sort.Slice(l, func(i, j int) bool {
+		// Since we are using bytes.Compare to compare TLV encodings we
+		// don't need to right pad s[i] and s[j] to the same length as
+		// suggested in X690. If len(s[i]) < len(s[j]) the length octet of
+		// s[i], which is the first determining byte, will inherently be
+		// smaller than the length octet of s[j]. This lets us skip the
+		// padding step.
+		return bytes.Compare(l[i], l[j]) < 0
+	})
+
+	var off int
+	for _, b := range l {
+		copy(dst[off:], b)
+		off += len(b)
+	}
+}
+
 type taggedEncoder struct {
 	// scratch contains temporary space for encoding the tag and length of
 	// an element in order to avoid extra allocations.
@@ -511,6 +555,9 @@ func makeBody(value reflect.Value, params fieldParameters) (e encoder, err error
 				}
 			}
 
+			if params.set {
+				return setEncoder(m), nil
+			}
 			return multiEncoder(m), nil
 		}
 	case reflect.String:
@@ -618,6 +665,15 @@ func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) {
 		tag = TagSet
 	}
 
+	// makeField can be called for a slice that should be treated as a SET
+	// but doesn't have params.set set, for instance when using a slice
+	// with the SET type name suffix. In this case getUniversalType returns
+	// TagSet, but makeBody doesn't know about that so will treat the slice
+	// as a sequence. To work around this we set params.set.
+	if tag == TagSet && !params.set {
+		params.set = true
+	}
+
 	t := new(taggedEncoder)
 
 	t.body, err = makeBody(v, params)
diff --git a/libgo/go/encoding/asn1/marshal_test.go b/libgo/go/encoding/asn1/marshal_test.go
index a77826a..5290522 100644
--- a/libgo/go/encoding/asn1/marshal_test.go
+++ b/libgo/go/encoding/asn1/marshal_test.go
@@ -319,3 +319,60 @@ func BenchmarkMarshal(b *testing.B) {
 		}
 	}
 }
+
+func TestSetEncoder(t *testing.T) {
+	testStruct := struct {
+		Strings []string `asn1:"set"`
+	}{
+		Strings: []string{"a", "aa", "b", "bb", "c", "cc"},
+	}
+
+	// Expected ordering of the SET should be:
+	// a, b, c, aa, bb, cc
+
+	output, err := Marshal(testStruct)
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+
+	expectedOrder := []string{"a", "b", "c", "aa", "bb", "cc"}
+	var resultStruct struct {
+		Strings []string `asn1:"set"`
+	}
+	rest, err := Unmarshal(output, &resultStruct)
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if len(rest) != 0 {
+		t.Error("Unmarshal returned extra garbage")
+	}
+	if !reflect.DeepEqual(expectedOrder, resultStruct.Strings) {
+		t.Errorf("Unexpected SET content. got: %s, want: %s", resultStruct.Strings, expectedOrder)
+	}
+}
+
+func TestSetEncoderSETSliceSuffix(t *testing.T) {
+	type testSetSET []string
+	testSet := testSetSET{"a", "aa", "b", "bb", "c", "cc"}
+
+	// Expected ordering of the SET should be:
+	// a, b, c, aa, bb, cc
+
+	output, err := Marshal(testSet)
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+
+	expectedOrder := testSetSET{"a", "b", "c", "aa", "bb", "cc"}
+	var resultSet testSetSET
+	rest, err := Unmarshal(output, &resultSet)
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if len(rest) != 0 {
+		t.Error("Unmarshal returned extra garbage")
+	}
+	if !reflect.DeepEqual(expectedOrder, resultSet) {
+		t.Errorf("Unexpected SET content. got: %s, want: %s", resultSet, expectedOrder)
+	}
+}
diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go
index 690d3ce..0c33f8e 100644
--- a/libgo/go/encoding/base64/base64.go
+++ b/libgo/go/encoding/base64/base64.go
@@ -480,15 +480,16 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err error) {
 
 	si := 0
 	for strconv.IntSize >= 64 && len(src)-si >= 8 && len(dst)-n >= 8 {
+		src2 := src[si : si+8]
 		if dn, ok := assemble64(
-			enc.decodeMap[src[si+0]],
-			enc.decodeMap[src[si+1]],
-			enc.decodeMap[src[si+2]],
-			enc.decodeMap[src[si+3]],
-			enc.decodeMap[src[si+4]],
-			enc.decodeMap[src[si+5]],
-			enc.decodeMap[src[si+6]],
-			enc.decodeMap[src[si+7]],
+			enc.decodeMap[src2[0]],
+			enc.decodeMap[src2[1]],
+			enc.decodeMap[src2[2]],
+			enc.decodeMap[src2[3]],
+			enc.decodeMap[src2[4]],
+			enc.decodeMap[src2[5]],
+			enc.decodeMap[src2[6]],
+			enc.decodeMap[src2[7]],
 		); ok {
 			binary.BigEndian.PutUint64(dst[n:], dn)
 			n += 6
@@ -504,11 +505,12 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err error) {
 	}
 
 	for len(src)-si >= 4 && len(dst)-n >= 4 {
+		src2 := src[si : si+4]
 		if dn, ok := assemble32(
-			enc.decodeMap[src[si+0]],
-			enc.decodeMap[src[si+1]],
-			enc.decodeMap[src[si+2]],
-			enc.decodeMap[src[si+3]],
+			enc.decodeMap[src2[0]],
+			enc.decodeMap[src2[1]],
+			enc.decodeMap[src2[2]],
+			enc.decodeMap[src2[3]],
 		); ok {
 			binary.BigEndian.PutUint32(dst[n:], dn)
 			n += 3
diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go
index bc67036..c2c9478 100644
--- a/libgo/go/encoding/base64/base64_test.go
+++ b/libgo/go/encoding/base64/base64_test.go
@@ -401,7 +401,7 @@ func TestDecoderIssue3577(t *testing.T) {
 		source: "VHdhcyBicmlsbGlnLCBhbmQgdGhlIHNsaXRoeSB0b3Zlcw==", // twas brillig...
 		nextc:  next,
 	})
-	errc := make(chan error)
+	errc := make(chan error, 1)
 	go func() {
 		_, err := ioutil.ReadAll(d)
 		errc <- err
diff --git a/libgo/go/encoding/binary/varint.go b/libgo/go/encoding/binary/varint.go
index bcb8ac9..38af610 100644
--- a/libgo/go/encoding/binary/varint.go
+++ b/libgo/go/encoding/binary/varint.go
@@ -106,13 +106,13 @@ var overflow = errors.New("binary: varint overflows a 64-bit integer")
 func ReadUvarint(r io.ByteReader) (uint64, error) {
 	var x uint64
 	var s uint
-	for i := 0; ; i++ {
+	for i := 0; i < MaxVarintLen64; i++ {
 		b, err := r.ReadByte()
 		if err != nil {
 			return x, err
 		}
 		if b < 0x80 {
-			if i > 9 || i == 9 && b > 1 {
+			if i == 9 && b > 1 {
 				return x, overflow
 			}
 			return x | uint64(b)<<s, nil
@@ -120,6 +120,7 @@ func ReadUvarint(r io.ByteReader) (uint64, error) {
 		x |= uint64(b&0x7f) << s
 		s += 7
 	}
+	return x, overflow
 }
 
 // ReadVarint reads an encoded signed integer from r and returns it as an int64.
diff --git a/libgo/go/encoding/binary/varint_test.go b/libgo/go/encoding/binary/varint_test.go
index ca411ec..6ef4c99 100644
--- a/libgo/go/encoding/binary/varint_test.go
+++ b/libgo/go/encoding/binary/varint_test.go
@@ -121,21 +121,27 @@ func TestBufferTooSmall(t *testing.T) {
 	}
 }
 
-func testOverflow(t *testing.T, buf []byte, n0 int, err0 error) {
+func testOverflow(t *testing.T, buf []byte, x0 uint64, n0 int, err0 error) {
 	x, n := Uvarint(buf)
 	if x != 0 || n != n0 {
 		t.Errorf("Uvarint(%v): got x = %d, n = %d; want 0, %d", buf, x, n, n0)
 	}
 
-	x, err := ReadUvarint(bytes.NewReader(buf))
-	if x != 0 || err != err0 {
-		t.Errorf("ReadUvarint(%v): got x = %d, err = %s; want 0, %s", buf, x, err, err0)
+	r := bytes.NewReader(buf)
+	len := r.Len()
+	x, err := ReadUvarint(r)
+	if x != x0 || err != err0 {
+		t.Errorf("ReadUvarint(%v): got x = %d, err = %s; want %d, %s", buf, x, err, x0, err0)
+	}
+	if read := len - r.Len(); read > MaxVarintLen64 {
+		t.Errorf("ReadUvarint(%v): read more than MaxVarintLen64 bytes, got %d", buf, read)
 	}
 }
 
 func TestOverflow(t *testing.T) {
-	testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x2}, -10, overflow)
-	testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1, 0, 0}, -13, overflow)
+	testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x2}, 0, -10, overflow)
+	testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1, 0, 0}, 0, -13, overflow)
+	testOverflow(t, []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 1<<64-1, 0, overflow) // 11 bytes, should overflow
 }
 
 func TestNonCanonicalZero(t *testing.T) {
diff --git a/libgo/go/encoding/csv/writer.go b/libgo/go/encoding/csv/writer.go
index 3f34bc5..ac64b4d 100644
--- a/libgo/go/encoding/csv/writer.go
+++ b/libgo/go/encoding/csv/writer.go
@@ -158,10 +158,24 @@ func (w *Writer) fieldNeedsQuotes(field string) bool {
 	if field == "" {
 		return false
 	}
-	if field == `\.` || strings.ContainsRune(field, w.Comma) || strings.ContainsAny(field, "\"\r\n") {
+
+	if field == `\.` {
 		return true
 	}
 
+	if w.Comma < utf8.RuneSelf {
+		for i := 0; i < len(field); i++ {
+			c := field[i]
+			if c == '\n' || c == '\r' || c == '"' || c == byte(w.Comma) {
+				return true
+			}
+		}
+	} else {
+		if strings.ContainsRune(field, w.Comma) || strings.ContainsAny(field, "\"\r\n") {
+			return true
+		}
+	}
+
 	r1, _ := utf8.DecodeRuneInString(field)
 	return unicode.IsSpace(r1)
 }
diff --git a/libgo/go/encoding/csv/writer_test.go b/libgo/go/encoding/csv/writer_test.go
index 011f01c..ab28b0d7 100644
--- a/libgo/go/encoding/csv/writer_test.go
+++ b/libgo/go/encoding/csv/writer_test.go
@@ -93,3 +93,20 @@ func TestError(t *testing.T) {
 		t.Error("Error should not be nil")
 	}
 }
+
+var benchmarkWriteData = [][]string{
+	{"abc", "def", "12356", "1234567890987654311234432141542132"},
+	{"abc", "def", "12356", "1234567890987654311234432141542132"},
+	{"abc", "def", "12356", "1234567890987654311234432141542132"},
+}
+
+func BenchmarkWrite(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		w := NewWriter(&bytes.Buffer{})
+		err := w.WriteAll(benchmarkWriteData)
+		if err != nil {
+			b.Fatal(err)
+		}
+		w.Flush()
+	}
+}
diff --git a/libgo/go/encoding/hex/hex_test.go b/libgo/go/encoding/hex/hex_test.go
index dbb00b9..31e3f68 100644
--- a/libgo/go/encoding/hex/hex_test.go
+++ b/libgo/go/encoding/hex/hex_test.go
@@ -267,7 +267,6 @@ func BenchmarkDecode(b *testing.B) {
 func BenchmarkDump(b *testing.B) {
 	for _, size := range []int{256, 1024, 4096, 16384} {
 		src := bytes.Repeat([]byte{2, 3, 5, 7, 9, 11, 13, 17}, size/8)
-		sink = make([]byte, 2*size)
 
 		b.Run(fmt.Sprintf("%v", size), func(b *testing.B) {
 			b.SetBytes(int64(size))
diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go
index b434846..86d8a69 100644
--- a/libgo/go/encoding/json/decode.go
+++ b/libgo/go/encoding/json/decode.go
@@ -213,9 +213,6 @@ type decodeState struct {
 	savedError            error
 	useNumber             bool
 	disallowUnknownFields bool
-	// safeUnquote is the number of current string literal bytes that don't
-	// need to be unquoted. When negative, no bytes need unquoting.
-	safeUnquote int
 }
 
 // readIndex returns the position of the last byte read.
@@ -317,27 +314,13 @@ func (d *decodeState) rescanLiteral() {
 Switch:
 	switch data[i-1] {
 	case '"': // string
-		// safeUnquote is initialized at -1, which means that all bytes
-		// checked so far can be unquoted at a later time with no work
-		// at all. When reaching the closing '"', if safeUnquote is
-		// still -1, all bytes can be unquoted with no work. Otherwise,
-		// only those bytes up until the first '\\' or non-ascii rune
-		// can be safely unquoted.
-		safeUnquote := -1
 		for ; i < len(data); i++ {
-			if c := data[i]; c == '\\' {
-				if safeUnquote < 0 { // first unsafe byte
-					safeUnquote = int(i - d.off)
-				}
+			switch data[i] {
+			case '\\':
 				i++ // escaped char
-			} else if c == '"' {
-				d.safeUnquote = safeUnquote
+			case '"':
 				i++ // tokenize the closing quote too
 				break Switch
-			} else if c >= utf8.RuneSelf {
-				if safeUnquote < 0 { // first unsafe byte
-					safeUnquote = int(i - d.off)
-				}
 			}
 		}
 	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': // number
@@ -691,7 +674,7 @@ func (d *decodeState) object(v reflect.Value) error {
 		start := d.readIndex()
 		d.rescanLiteral()
 		item := d.data[start:d.readIndex()]
-		key, ok := d.unquoteBytes(item)
+		key, ok := unquoteBytes(item)
 		if !ok {
 			panic(phasePanicMsg)
 		}
@@ -892,7 +875,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool
 			d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.readIndex())})
 			return nil
 		}
-		s, ok := d.unquoteBytes(item)
+		s, ok := unquoteBytes(item)
 		if !ok {
 			if fromQuoted {
 				return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())
@@ -943,7 +926,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool
 		}
 
 	case '"': // string
-		s, ok := d.unquoteBytes(item)
+		s, ok := unquoteBytes(item)
 		if !ok {
 			if fromQuoted {
 				return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())
@@ -1103,7 +1086,7 @@ func (d *decodeState) objectInterface() map[string]interface{} {
 		start := d.readIndex()
 		d.rescanLiteral()
 		item := d.data[start:d.readIndex()]
-		key, ok := d.unquote(item)
+		key, ok := unquote(item)
 		if !ok {
 			panic(phasePanicMsg)
 		}
@@ -1152,7 +1135,7 @@ func (d *decodeState) literalInterface() interface{} {
 		return c == 't'
 
 	case '"': // string
-		s, ok := d.unquote(item)
+		s, ok := unquote(item)
 		if !ok {
 			panic(phasePanicMsg)
 		}
@@ -1195,26 +1178,38 @@ func getu4(s []byte) rune {
 
 // unquote converts a quoted JSON string literal s into an actual string t.
 // The rules are different than for Go, so cannot use strconv.Unquote.
-// The first byte in s must be '"'.
-func (d *decodeState) unquote(s []byte) (t string, ok bool) {
-	s, ok = d.unquoteBytes(s)
+func unquote(s []byte) (t string, ok bool) {
+	s, ok = unquoteBytes(s)
 	t = string(s)
 	return
 }
 
-func (d *decodeState) unquoteBytes(s []byte) (t []byte, ok bool) {
-	// We already know that s[0] == '"'. However, we don't know that the
-	// closing quote exists in all cases, such as when the string is nested
-	// via the ",string" option.
-	if len(s) < 2 || s[len(s)-1] != '"' {
+func unquoteBytes(s []byte) (t []byte, ok bool) {
+	if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {
 		return
 	}
 	s = s[1 : len(s)-1]
 
-	// If there are no unusual characters, no unquoting is needed, so return
-	// a slice of the original bytes.
-	r := d.safeUnquote
-	if r == -1 {
+	// Check for unusual characters. If there are none,
+	// then no unquoting is needed, so return a slice of the
+	// original bytes.
+	r := 0
+	for r < len(s) {
+		c := s[r]
+		if c == '\\' || c == '"' || c < ' ' {
+			break
+		}
+		if c < utf8.RuneSelf {
+			r++
+			continue
+		}
+		rr, size := utf8.DecodeRune(s[r:])
+		if rr == utf8.RuneError && size == 1 {
+			break
+		}
+		r += size
+	}
+	if r == len(s) {
 		return s, true
 	}
 
diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go
index 498bd97..219e845 100644
--- a/libgo/go/encoding/json/decode_test.go
+++ b/libgo/go/encoding/json/decode_test.go
@@ -2419,7 +2419,7 @@ func (m *textUnmarshalerString) UnmarshalText(text []byte) error {
 	return nil
 }
 
-// Test unmarshal to a map, with map key is a user defined type.
+// Test unmarshal to a map, where the map key is a user defined type.
 // See golang.org/issues/34437.
 func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) {
 	var p map[textUnmarshalerString]string
@@ -2428,6 +2428,147 @@ func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) {
 	}
 
 	if _, ok := p["foo"]; !ok {
-		t.Errorf(`Key "foo" is not existed in map: %v`, p)
+		t.Errorf(`Key "foo" does not exist in map: %v`, p)
+	}
+}
+
+func TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) {
+	// See golang.org/issues/38105.
+	var p map[textUnmarshalerString]string
+	if err := Unmarshal([]byte(`{"开源":"12345开源"}`), &p); err != nil {
+		t.Fatalf("Unmarshal unexpected error: %v", err)
+	}
+	if _, ok := p["开源"]; !ok {
+		t.Errorf(`Key "开源" does not exist in map: %v`, p)
+	}
+
+	// See golang.org/issues/38126.
+	type T struct {
+		F1 string `json:"F1,string"`
+	}
+	t1 := T{"aaa\tbbb"}
+
+	b, err := Marshal(t1)
+	if err != nil {
+		t.Fatalf("Marshal unexpected error: %v", err)
+	}
+	var t2 T
+	if err := Unmarshal(b, &t2); err != nil {
+		t.Fatalf("Unmarshal unexpected error: %v", err)
+	}
+	if t1 != t2 {
+		t.Errorf("Marshal and Unmarshal roundtrip mismatch: want %q got %q", t1, t2)
+	}
+
+	// See golang.org/issues/39555.
+	input := map[textUnmarshalerString]string{"FOO": "", `"`: ""}
+
+	encoded, err := Marshal(input)
+	if err != nil {
+		t.Fatalf("Marshal unexpected error: %v", err)
+	}
+	var got map[textUnmarshalerString]string
+	if err := Unmarshal(encoded, &got); err != nil {
+		t.Fatalf("Unmarshal unexpected error: %v", err)
+	}
+	want := map[textUnmarshalerString]string{"foo": "", `"`: ""}
+	if !reflect.DeepEqual(want, got) {
+		t.Fatalf("Unexpected roundtrip result:\nwant: %q\ngot:  %q", want, got)
+	}
+}
+
+func TestUnmarshalMaxDepth(t *testing.T) {
+	testcases := []struct {
+		name        string
+		data        string
+		errMaxDepth bool
+	}{
+		{
+			name:        "ArrayUnderMaxNestingDepth",
+			data:        `{"a":` + strings.Repeat(`[`, 10000-1) + strings.Repeat(`]`, 10000-1) + `}`,
+			errMaxDepth: false,
+		},
+		{
+			name:        "ArrayOverMaxNestingDepth",
+			data:        `{"a":` + strings.Repeat(`[`, 10000) + strings.Repeat(`]`, 10000) + `}`,
+			errMaxDepth: true,
+		},
+		{
+			name:        "ArrayOverStackDepth",
+			data:        `{"a":` + strings.Repeat(`[`, 3000000) + strings.Repeat(`]`, 3000000) + `}`,
+			errMaxDepth: true,
+		},
+		{
+			name:        "ObjectUnderMaxNestingDepth",
+			data:        `{"a":` + strings.Repeat(`{"a":`, 10000-1) + `0` + strings.Repeat(`}`, 10000-1) + `}`,
+			errMaxDepth: false,
+		},
+		{
+			name:        "ObjectOverMaxNestingDepth",
+			data:        `{"a":` + strings.Repeat(`{"a":`, 10000) + `0` + strings.Repeat(`}`, 10000) + `}`,
+			errMaxDepth: true,
+		},
+		{
+			name:        "ObjectOverStackDepth",
+			data:        `{"a":` + strings.Repeat(`{"a":`, 3000000) + `0` + strings.Repeat(`}`, 3000000) + `}`,
+			errMaxDepth: true,
+		},
+	}
+
+	targets := []struct {
+		name     string
+		newValue func() interface{}
+	}{
+		{
+			name: "unstructured",
+			newValue: func() interface{} {
+				var v interface{}
+				return &v
+			},
+		},
+		{
+			name: "typed named field",
+			newValue: func() interface{} {
+				v := struct {
+					A interface{} `json:"a"`
+				}{}
+				return &v
+			},
+		},
+		{
+			name: "typed missing field",
+			newValue: func() interface{} {
+				v := struct {
+					B interface{} `json:"b"`
+				}{}
+				return &v
+			},
+		},
+		{
+			name: "custom unmarshaler",
+			newValue: func() interface{} {
+				v := unmarshaler{}
+				return &v
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		for _, target := range targets {
+			t.Run(target.name+"-"+tc.name, func(t *testing.T) {
+				err := Unmarshal([]byte(tc.data), target.newValue())
+				if !tc.errMaxDepth {
+					if err != nil {
+						t.Errorf("unexpected error: %v", err)
+					}
+				} else {
+					if err == nil {
+						t.Errorf("expected error containing 'exceeded max depth', got none")
+					} else if !strings.Contains(err.Error(), "exceeded max depth") {
+						t.Errorf("expected error containing 'exceeded max depth', got: %v", err)
+					}
+				}
+			})
+		}
 	}
 }
diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go
index 39cdaeb..578d551 100644
--- a/libgo/go/encoding/json/encode.go
+++ b/libgo/go/encoding/json/encode.go
@@ -635,11 +635,12 @@ func stringEncoder(e *encodeState, v reflect.Value, opts encOpts) {
 		return
 	}
 	if opts.quoted {
-		b := make([]byte, 0, v.Len()+2)
-		b = append(b, '"')
-		b = append(b, []byte(v.String())...)
-		b = append(b, '"')
-		e.stringBytes(b, opts.escapeHTML)
+		e2 := newEncodeState()
+		// Since we encode the string twice, we only need to escape HTML
+		// the first time.
+		e2.string(v.String(), opts.escapeHTML)
+		e.stringBytes(e2.Bytes(), false)
+		encodeStatePool.Put(e2)
 	} else {
 		e.string(v.String(), opts.escapeHTML)
 	}
@@ -649,7 +650,7 @@ func stringEncoder(e *encodeState, v reflect.Value, opts encOpts) {
 func isValidNumber(s string) bool {
 	// This function implements the JSON numbers grammar.
 	// See https://tools.ietf.org/html/rfc7159#section-6
-	// and https://json.org/number.gif
+	// and https://www.json.org/img/number.png
 
 	if s == "" {
 		return false
diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go
index 5110c7d..7290eca 100644
--- a/libgo/go/encoding/json/encode_test.go
+++ b/libgo/go/encoding/json/encode_test.go
@@ -79,37 +79,66 @@ type StringTag struct {
 	NumberStr  Number  `json:",string"`
 }
 
-var stringTagExpected = `{
- "BoolStr": "true",
- "IntStr": "42",
- "UintptrStr": "44",
- "StrStr": "\"xzbit\"",
- "NumberStr": "46"
-}`
-
-func TestStringTag(t *testing.T) {
-	var s StringTag
-	s.BoolStr = true
-	s.IntStr = 42
-	s.UintptrStr = 44
-	s.StrStr = "xzbit"
-	s.NumberStr = "46"
-	got, err := MarshalIndent(&s, "", " ")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got := string(got); got != stringTagExpected {
-		t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected)
+func TestRoundtripStringTag(t *testing.T) {
+	tests := []struct {
+		name string
+		in   StringTag
+		want string // empty to just test that we roundtrip
+	}{
+		{
+			name: "AllTypes",
+			in: StringTag{
+				BoolStr:    true,
+				IntStr:     42,
+				UintptrStr: 44,
+				StrStr:     "xzbit",
+				NumberStr:  "46",
+			},
+			want: `{
+				"BoolStr": "true",
+				"IntStr": "42",
+				"UintptrStr": "44",
+				"StrStr": "\"xzbit\"",
+				"NumberStr": "46"
+			}`,
+		},
+		{
+			// See golang.org/issues/38173.
+			name: "StringDoubleEscapes",
+			in: StringTag{
+				StrStr:    "\b\f\n\r\t\"\\",
+				NumberStr: "0", // just to satisfy the roundtrip
+			},
+			want: `{
+				"BoolStr": "false",
+				"IntStr": "0",
+				"UintptrStr": "0",
+				"StrStr": "\"\\u0008\\u000c\\n\\r\\t\\\"\\\\\"",
+				"NumberStr": "0"
+			}`,
+		},
 	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// Indent with a tab prefix to make the multi-line string
+			// literals in the table nicer to read.
+			got, err := MarshalIndent(&test.in, "\t\t\t", "\t")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := string(got); got != test.want {
+				t.Fatalf(" got: %s\nwant: %s\n", got, test.want)
+			}
 
-	// Verify that it round-trips.
-	var s2 StringTag
-	err = NewDecoder(bytes.NewReader(got)).Decode(&s2)
-	if err != nil {
-		t.Fatalf("Decode: %v", err)
-	}
-	if !reflect.DeepEqual(s, s2) {
-		t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", s, string(got), s2)
+			// Verify that it round-trips.
+			var s2 StringTag
+			if err := Unmarshal(got, &s2); err != nil {
+				t.Fatalf("Decode: %v", err)
+			}
+			if !reflect.DeepEqual(test.in, s2) {
+				t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", test.in, string(got), s2)
+			}
+		})
 	}
 }
 
diff --git a/libgo/go/encoding/json/scanner.go b/libgo/go/encoding/json/scanner.go
index 552bd70..9dc1903 100644
--- a/libgo/go/encoding/json/scanner.go
+++ b/libgo/go/encoding/json/scanner.go
@@ -139,6 +139,10 @@ const (
 	parseArrayValue         // parsing array value
 )
 
+// This limits the max nesting depth to prevent stack overflow.
+// This is permitted by https://tools.ietf.org/html/rfc7159#section-9
+const maxNestingDepth = 10000
+
 // reset prepares the scanner for use.
 // It must be called before calling s.step.
 func (s *scanner) reset() {
@@ -168,8 +172,13 @@ func (s *scanner) eof() int {
 }
 
 // pushParseState pushes a new parse state p onto the parse stack.
-func (s *scanner) pushParseState(p int) {
-	s.parseState = append(s.parseState, p)
+// an error state is returned if maxNestingDepth was exceeded, otherwise successState is returned.
+func (s *scanner) pushParseState(c byte, newParseState int, successState int) int {
+	s.parseState = append(s.parseState, newParseState)
+	if len(s.parseState) <= maxNestingDepth {
+		return successState
+	}
+	return s.error(c, "exceeded max depth")
 }
 
 // popParseState pops a parse state (already obtained) off the stack
@@ -186,12 +195,12 @@ func (s *scanner) popParseState() {
 }
 
 func isSpace(c byte) bool {
-	return c == ' ' || c == '\t' || c == '\r' || c == '\n'
+	return c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n')
 }
 
 // stateBeginValueOrEmpty is the state after reading `[`.
 func stateBeginValueOrEmpty(s *scanner, c byte) int {
-	if c <= ' ' && isSpace(c) {
+	if isSpace(c) {
 		return scanSkipSpace
 	}
 	if c == ']' {
@@ -202,18 +211,16 @@ func stateBeginValueOrEmpty(s *scanner, c byte) int {
 
 // stateBeginValue is the state at the beginning of the input.
 func stateBeginValue(s *scanner, c byte) int {
-	if c <= ' ' && isSpace(c) {
+	if isSpace(c) {
 		return scanSkipSpace
 	}
 	switch c {
 	case '{':
 		s.step = stateBeginStringOrEmpty
-		s.pushParseState(parseObjectKey)
-		return scanBeginObject
+		return s.pushParseState(c, parseObjectKey, scanBeginObject)
 	case '[':
 		s.step = stateBeginValueOrEmpty
-		s.pushParseState(parseArrayValue)
-		return scanBeginArray
+		return s.pushParseState(c, parseArrayValue, scanBeginArray)
 	case '"':
 		s.step = stateInString
 		return scanBeginLiteral
@@ -242,7 +249,7 @@ func stateBeginValue(s *scanner, c byte) int {
 
 // stateBeginStringOrEmpty is the state after reading `{`.
 func stateBeginStringOrEmpty(s *scanner, c byte) int {
-	if c <= ' ' && isSpace(c) {
+	if isSpace(c) {
 		return scanSkipSpace
 	}
 	if c == '}' {
@@ -255,7 +262,7 @@ func stateBeginStringOrEmpty(s *scanner, c byte) int {
 
 // stateBeginString is the state after reading `{"key": value,`.
 func stateBeginString(s *scanner, c byte) int {
-	if c <= ' ' && isSpace(c) {
+	if isSpace(c) {
 		return scanSkipSpace
 	}
 	if c == '"' {
@@ -275,7 +282,7 @@ func stateEndValue(s *scanner, c byte) int {
 		s.endTop = true
 		return stateEndTop(s, c)
 	}
-	if c <= ' ' && isSpace(c) {
+	if isSpace(c) {
 		s.step = stateEndValue
 		return scanSkipSpace
 	}
diff --git a/libgo/go/encoding/json/stream_test.go b/libgo/go/encoding/json/stream_test.go
index ebb4f23..c9e5334 100644
--- a/libgo/go/encoding/json/stream_test.go
+++ b/libgo/go/encoding/json/stream_test.go
@@ -144,14 +144,15 @@ func TestEncoderSetEscapeHTML(t *testing.T) {
 		},
 		{
 			"stringOption", stringOption,
-			`{"bar":"\"\u003chtml\u003efoobar\u003c/html\u003e\""}`,
+			`{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`,
 			`{"bar":"\"<html>foobar</html>\""}`,
 		},
 	} {
 		var buf bytes.Buffer
 		enc := NewEncoder(&buf)
 		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("Encode(%s): %s", tt.name, err)
+			t.Errorf("Encode(%s): %s", tt.name, err)
+			continue
 		}
 		if got := strings.TrimSpace(buf.String()); got != tt.wantEscape {
 			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape)
@@ -159,7 +160,8 @@ func TestEncoderSetEscapeHTML(t *testing.T) {
 		buf.Reset()
 		enc.SetEscapeHTML(false)
 		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err)
+			t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err)
+			continue
 		}
 		if got := strings.TrimSpace(buf.String()); got != tt.want {
 			t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q",
diff --git a/libgo/go/encoding/xml/marshal.go b/libgo/go/encoding/xml/marshal.go
index e325f31..0554b0d 100644
--- a/libgo/go/encoding/xml/marshal.go
+++ b/libgo/go/encoding/xml/marshal.go
@@ -479,8 +479,11 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat
 		xmlname := tinfo.xmlname
 		if xmlname.name != "" {
 			start.Name.Space, start.Name.Local = xmlname.xmlns, xmlname.name
-		} else if v, ok := xmlname.value(val).Interface().(Name); ok && v.Local != "" {
-			start.Name = v
+		} else {
+			fv := xmlname.value(val, dontInitNilPointers)
+			if v, ok := fv.Interface().(Name); ok && v.Local != "" {
+				start.Name = v
+			}
 		}
 	}
 	if start.Name.Local == "" && finfo != nil {
@@ -500,7 +503,7 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat
 		if finfo.flags&fAttr == 0 {
 			continue
 		}
-		fv := finfo.value(val)
+		fv := finfo.value(val, dontInitNilPointers)
 
 		if finfo.flags&fOmitEmpty != 0 && isEmptyValue(fv) {
 			continue
@@ -803,7 +806,12 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error {
 		if finfo.flags&fAttr != 0 {
 			continue
 		}
-		vf := finfo.value(val)
+		vf := finfo.value(val, dontInitNilPointers)
+		if !vf.IsValid() {
+			// The field is behind an anonymous struct field that's
+			// nil. Skip it.
+			continue
+		}
 
 		switch finfo.flags & fMode {
 		case fCDATA, fCharData:
diff --git a/libgo/go/encoding/xml/marshal_test.go b/libgo/go/encoding/xml/marshal_test.go
index 8473158b..31309ef 100644
--- a/libgo/go/encoding/xml/marshal_test.go
+++ b/libgo/go/encoding/xml/marshal_test.go
@@ -309,6 +309,11 @@ type ChardataEmptyTest struct {
 	Contents *string `xml:",chardata"`
 }
 
+type PointerAnonFields struct {
+	*MyInt
+	*NamedType
+}
+
 type MyMarshalerTest struct {
 }
 
@@ -889,6 +894,18 @@ var marshalTests = []struct {
 			`</EmbedA>`,
 	},
 
+	// Anonymous struct pointer field which is nil
+	{
+		Value:     &EmbedB{},
+		ExpectXML: `<EmbedB><FieldB></FieldB></EmbedB>`,
+	},
+
+	// Other kinds of nil anonymous fields
+	{
+		Value:     &PointerAnonFields{},
+		ExpectXML: `<PointerAnonFields></PointerAnonFields>`,
+	},
+
 	// Test that name casing matters
 	{
 		Value:     &NameCasing{Xy: "mixed", XY: "upper", XyA: "mixedA", XYA: "upperA"},
diff --git a/libgo/go/encoding/xml/read.go b/libgo/go/encoding/xml/read.go
index 10a60ee..ef5df3f 100644
--- a/libgo/go/encoding/xml/read.go
+++ b/libgo/go/encoding/xml/read.go
@@ -435,7 +435,7 @@ func (d *Decoder) unmarshal(val reflect.Value, start *StartElement) error {
 				}
 				return UnmarshalError(e)
 			}
-			fv := finfo.value(sv)
+			fv := finfo.value(sv, initNilPointers)
 			if _, ok := fv.Interface().(Name); ok {
 				fv.Set(reflect.ValueOf(start.Name))
 			}
@@ -449,7 +449,7 @@ func (d *Decoder) unmarshal(val reflect.Value, start *StartElement) error {
 				finfo := &tinfo.fields[i]
 				switch finfo.flags & fMode {
 				case fAttr:
-					strv := finfo.value(sv)
+					strv := finfo.value(sv, initNilPointers)
 					if a.Name.Local == finfo.name && (finfo.xmlns == "" || finfo.xmlns == a.Name.Space) {
 						if err := d.unmarshalAttr(strv, a); err != nil {
 							return err
@@ -465,7 +465,7 @@ func (d *Decoder) unmarshal(val reflect.Value, start *StartElement) error {
 			}
 			if !handled && any >= 0 {
 				finfo := &tinfo.fields[any]
-				strv := finfo.value(sv)
+				strv := finfo.value(sv, initNilPointers)
 				if err := d.unmarshalAttr(strv, a); err != nil {
 					return err
 				}
@@ -478,22 +478,22 @@ func (d *Decoder) unmarshal(val reflect.Value, start *StartElement) error {
 			switch finfo.flags & fMode {
 			case fCDATA, fCharData:
 				if !saveData.IsValid() {
-					saveData = finfo.value(sv)
+					saveData = finfo.value(sv, initNilPointers)
 				}
 
 			case fComment:
 				if !saveComment.IsValid() {
-					saveComment = finfo.value(sv)
+					saveComment = finfo.value(sv, initNilPointers)
 				}
 
 			case fAny, fAny | fElement:
 				if !saveAny.IsValid() {
-					saveAny = finfo.value(sv)
+					saveAny = finfo.value(sv, initNilPointers)
 				}
 
 			case fInnerXML:
 				if !saveXML.IsValid() {
-					saveXML = finfo.value(sv)
+					saveXML = finfo.value(sv, initNilPointers)
 					if d.saved == nil {
 						saveXMLIndex = 0
 						d.saved = new(bytes.Buffer)
@@ -687,7 +687,7 @@ Loop:
 		}
 		if len(finfo.parents) == len(parents) && finfo.name == start.Name.Local {
 			// It's a perfect match, unmarshal the field.
-			return true, d.unmarshal(finfo.value(sv), start)
+			return true, d.unmarshal(finfo.value(sv, initNilPointers), start)
 		}
 		if len(finfo.parents) > len(parents) && finfo.parents[len(parents)] == start.Name.Local {
 			// It's a prefix for the field. Break and recurse
diff --git a/libgo/go/encoding/xml/typeinfo.go b/libgo/go/encoding/xml/typeinfo.go
index 639952c..f30fe58 100644
--- a/libgo/go/encoding/xml/typeinfo.go
+++ b/libgo/go/encoding/xml/typeinfo.go
@@ -344,15 +344,25 @@ func (e *TagPathError) Error() string {
 	return fmt.Sprintf("%s field %q with tag %q conflicts with field %q with tag %q", e.Struct, e.Field1, e.Tag1, e.Field2, e.Tag2)
 }
 
+const (
+	initNilPointers     = true
+	dontInitNilPointers = false
+)
+
 // value returns v's field value corresponding to finfo.
-// It's equivalent to v.FieldByIndex(finfo.idx), but initializes
-// and dereferences pointers as necessary.
-func (finfo *fieldInfo) value(v reflect.Value) reflect.Value {
+// It's equivalent to v.FieldByIndex(finfo.idx), but when passed
+// initNilPointers, it initializes and dereferences pointers as necessary.
+// When passed dontInitNilPointers and a nil pointer is reached, the function
+// returns a zero reflect.Value.
+func (finfo *fieldInfo) value(v reflect.Value, shouldInitNilPointers bool) reflect.Value {
 	for i, x := range finfo.idx {
 		if i > 0 {
 			t := v.Type()
 			if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct {
 				if v.IsNil() {
+					if !shouldInitNilPointers {
+						return reflect.Value{}
+					}
 					v.Set(reflect.New(v.Type().Elem()))
 				}
 				v = v.Elem()
diff --git a/libgo/go/encoding/xml/xml.go b/libgo/go/encoding/xml/xml.go
index 5e73dcf..adaf4da 100644
--- a/libgo/go/encoding/xml/xml.go
+++ b/libgo/go/encoding/xml/xml.go
@@ -960,7 +960,7 @@ func (d *Decoder) ungetc(b byte) {
 	d.offset--
 }
 
-var entity = map[string]int{
+var entity = map[string]rune{
 	"lt":   '<',
 	"gt":   '>',
 	"amp":  '&',
@@ -1055,7 +1055,7 @@ Input:
 					d.buf.WriteByte(';')
 					n, err := strconv.ParseUint(s, base, 64)
 					if err == nil && n <= unicode.MaxRune {
-						text = string(n)
+						text = string(rune(n))
 						haveText = true
 					}
 				}