diff options
author | Ian Lance Taylor <ian@gcc.gnu.org> | 2015-10-31 00:59:47 +0000 |
---|---|---|
committer | Ian Lance Taylor <ian@gcc.gnu.org> | 2015-10-31 00:59:47 +0000 |
commit | af146490bb04205107cb23e301ec7a8ff927b5fc (patch) | |
tree | 13beeaed3698c61903fe93fb1ce70bd9b18d4e7f /libgo/go/text | |
parent | 725e1be3406315d9bcc8195d7eef0a7082b3c7cc (diff) | |
download | gcc-af146490bb04205107cb23e301ec7a8ff927b5fc.zip gcc-af146490bb04205107cb23e301ec7a8ff927b5fc.tar.gz gcc-af146490bb04205107cb23e301ec7a8ff927b5fc.tar.bz2 |
runtime: Remove now unnecessary pad field from ParFor.
It is not needed due to the removal of the ctx field.
Reviewed-on: https://go-review.googlesource.com/16525
From-SVN: r229616
Diffstat (limited to 'libgo/go/text')
-rw-r--r-- | libgo/go/text/scanner/example_test.go | 40 | ||||
-rw-r--r-- | libgo/go/text/scanner/scanner.go | 23 | ||||
-rw-r--r-- | libgo/go/text/scanner/scanner_test.go | 49 | ||||
-rw-r--r-- | libgo/go/text/template/doc.go | 4 | ||||
-rw-r--r-- | libgo/go/text/template/exec.go | 77 | ||||
-rw-r--r-- | libgo/go/text/template/exec_test.go | 101 | ||||
-rw-r--r-- | libgo/go/text/template/funcs.go | 4 | ||||
-rw-r--r-- | libgo/go/text/template/helper.go | 11 | ||||
-rw-r--r-- | libgo/go/text/template/multi_test.go | 73 | ||||
-rw-r--r-- | libgo/go/text/template/option.go | 74 | ||||
-rw-r--r-- | libgo/go/text/template/parse/lex.go | 13 | ||||
-rw-r--r-- | libgo/go/text/template/parse/lex_test.go | 57 | ||||
-rw-r--r-- | libgo/go/text/template/parse/node.go | 15 | ||||
-rw-r--r-- | libgo/go/text/template/parse/parse.go | 35 | ||||
-rw-r--r-- | libgo/go/text/template/parse/parse_test.go | 31 | ||||
-rw-r--r-- | libgo/go/text/template/template.go | 84 |
16 files changed, 584 insertions, 107 deletions
diff --git a/libgo/go/text/scanner/example_test.go b/libgo/go/text/scanner/example_test.go new file mode 100644 index 0000000..f8b51b7 --- /dev/null +++ b/libgo/go/text/scanner/example_test.go @@ -0,0 +1,40 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package scanner_test + +import ( + "fmt" + "strings" + "text/scanner" +) + +func Example() { + const src = ` + // This is scanned code. + if a > 10 { + someParsable = text + }` + var s scanner.Scanner + s.Init(strings.NewReader(src)) + var tok rune + for tok != scanner.EOF { + tok = s.Scan() + fmt.Println("At position", s.Pos(), ":", s.TokenText()) + } + + // Output: + // At position 3:4 : if + // At position 3:6 : a + // At position 3:8 : > + // At position 3:11 : 10 + // At position 3:13 : { + // At position 4:15 : someParsable + // At position 4:17 : = + // At position 4:22 : text + // At position 5:3 : } + // At position 5:3 : +} diff --git a/libgo/go/text/scanner/scanner.go b/libgo/go/text/scanner/scanner.go index 5199ee4..3ab01ed 100644 --- a/libgo/go/text/scanner/scanner.go +++ b/libgo/go/text/scanner/scanner.go @@ -12,17 +12,6 @@ // literals as defined by the Go language specification. It may be // customized to recognize only a subset of those literals and to recognize // different identifier and white space characters. -// -// Basic usage pattern: -// -// var s scanner.Scanner -// s.Init(src) -// tok := s.Scan() -// for tok != scanner.EOF { -// // do something with tok -// tok = s.Scan() -// } -// package scanner import ( @@ -43,7 +32,7 @@ type Position struct { Column int // column number, starting at 1 (character count per line) } -// IsValid returns true if the position is valid. +// IsValid reports whether the position is valid. func (pos *Position) IsValid() bool { return pos.Line > 0 } func (pos Position) String() string { @@ -208,7 +197,7 @@ func (s *Scanner) Init(src io.Reader) *Scanner { s.tokPos = -1 // initialize one character look-ahead - s.ch = -1 // no char read yet + s.ch = -2 // no char read yet, not EOF // initialize public fields s.Error = nil @@ -314,7 +303,9 @@ func (s *Scanner) Next() rune { s.tokPos = -1 // don't collect token text s.Line = 0 // invalidate token position ch := s.Peek() - s.ch = s.next() + if ch != EOF { + s.ch = s.next() + } return ch } @@ -322,7 +313,7 @@ func (s *Scanner) Next() rune { // the scanner. It returns EOF if the scanner's position is at the last // character of the source. func (s *Scanner) Peek() rune { - if s.ch < 0 { + if s.ch == -2 { // this code is only run for the very first character s.ch = s.next() if s.ch == '\uFEFF' { @@ -597,6 +588,8 @@ redo: } default: switch ch { + case EOF: + break case '"': if s.Mode&ScanStrings != 0 { s.scanString('"') diff --git a/libgo/go/text/scanner/scanner_test.go b/libgo/go/text/scanner/scanner_test.go index 702fac2..798bed7 100644 --- a/libgo/go/text/scanner/scanner_test.go +++ b/libgo/go/text/scanner/scanner_test.go @@ -616,3 +616,52 @@ func TestPos(t *testing.T) { t.Errorf("%d errors", s.ErrorCount) } } + +type countReader int + +func (r *countReader) Read([]byte) (int, error) { + *r++ + return 0, io.EOF +} + +func TestNextEOFHandling(t *testing.T) { + var r countReader + + // corner case: empty source + s := new(Scanner).Init(&r) + + tok := s.Next() + if tok != EOF { + t.Error("1) EOF not reported") + } + + tok = s.Peek() + if tok != EOF { + t.Error("2) EOF not reported") + } + + if r != 1 { + t.Errorf("scanner called Read %d times, not once", r) + } +} + +func TestScanEOFHandling(t *testing.T) { + var r countReader + + // corner case: empty source + s := new(Scanner).Init(&r) + + tok := s.Scan() + if tok != EOF { + t.Error("1) EOF not reported") + } + + tok = s.Peek() + if tok != EOF { + t.Error("2) EOF not reported") + } + + if r != 1 { + t.Errorf("scanner called Read %d times, not once", r) + } +} diff --git a/libgo/go/text/template/doc.go b/libgo/go/text/template/doc.go index 223c595..0ce63f6 100644 --- a/libgo/go/text/template/doc.go +++ b/libgo/go/text/template/doc.go @@ -18,7 +18,7 @@ structure as execution proceeds. The input text for a template is UTF-8-encoded text in any format. "Actions"--data evaluations or control structures--are delimited by "{{" and "}}"; all text outside actions is copied to the output unchanged. -Actions may not span newlines, although comments can. +Except for raw strings, actions may not span newlines, although comments can. Once parsed, a template may be executed safely in parallel. @@ -106,7 +106,7 @@ An argument is a simple value, denoted by one of the following. - A boolean, string, character, integer, floating-point, imaginary or complex constant in Go syntax. These behave like Go's untyped - constants, although raw strings may not span newlines. + constants. - The keyword nil, representing an untyped Go nil. - The character '.' (period): . diff --git a/libgo/go/text/template/exec.go b/libgo/go/text/template/exec.go index b00e10c..daba788 100644 --- a/libgo/go/text/template/exec.go +++ b/libgo/go/text/template/exec.go @@ -113,7 +113,10 @@ func errRecover(errp *error) { // the output writer. // A template may be executed safely in parallel. func (t *Template) ExecuteTemplate(wr io.Writer, name string, data interface{}) error { - tmpl := t.tmpl[name] + var tmpl *Template + if t.common != nil { + tmpl = t.tmpl[name] + } if tmpl == nil { return fmt.Errorf("template: no template %q associated with template %q", name, t.name) } @@ -134,26 +137,36 @@ func (t *Template) Execute(wr io.Writer, data interface{}) (err error) { wr: wr, vars: []variable{{"$", value}}, } - t.init() if t.Tree == nil || t.Root == nil { - var b bytes.Buffer - for name, tmpl := range t.tmpl { - if tmpl.Tree == nil || tmpl.Root == nil { - continue - } - if b.Len() > 0 { - b.WriteString(", ") - } - fmt.Fprintf(&b, "%q", name) + state.errorf("%q is an incomplete or empty template%s", t.Name(), t.DefinedTemplates()) + } + state.walk(value, t.Root) + return +} + +// DefinedTemplates returns a string listing the defined templates, +// prefixed by the string "defined templates are: ". If there are none, +// it returns the empty string. For generating an error message here +// and in html/template. +func (t *Template) DefinedTemplates() string { + if t.common == nil { + return "" + } + var b bytes.Buffer + for name, tmpl := range t.tmpl { + if tmpl.Tree == nil || tmpl.Root == nil { + continue } - var s string if b.Len() > 0 { - s = "; defined templates are: " + b.String() + b.WriteString(", ") } - state.errorf("%q is an incomplete or empty template%s", t.Name(), s) + fmt.Fprintf(&b, "%q", name) } - state.walk(value, t.Root) - return + var s string + if b.Len() > 0 { + s = "; defined templates are: " + b.String() + } + return s } // Walk functions step through the major pieces of the template structure, @@ -418,11 +431,14 @@ func (s *state) evalFieldNode(dot reflect.Value, field *parse.FieldNode, args [] func (s *state) evalChainNode(dot reflect.Value, chain *parse.ChainNode, args []parse.Node, final reflect.Value) reflect.Value { s.at(chain) - // (pipe).Field1.Field2 has pipe as .Node, fields as .Field. Eval the pipeline, then the fields. - pipe := s.evalArg(dot, nil, chain.Node) if len(chain.Field) == 0 { s.errorf("internal error: no fields in evalChainNode") } + if chain.Node.Type() == parse.NodeNil { + s.errorf("indirection through explicit nil in %s", chain) + } + // (pipe).Field1.Field2 has pipe as .Node, fields as .Field. Eval the pipeline, then the fields. + pipe := s.evalArg(dot, nil, chain.Node) return s.evalFieldChain(dot, pipe, chain, chain.Field, args, final) } @@ -505,7 +521,18 @@ func (s *state) evalField(dot reflect.Value, fieldName string, node parse.Node, if hasArgs { s.errorf("%s is not a method but has arguments", fieldName) } - return receiver.MapIndex(nameVal) + result := receiver.MapIndex(nameVal) + if !result.IsValid() { + switch s.tmpl.option.missingKey { + case mapInvalid: + // Just use the invalid value. + case mapZeroValue: + result = reflect.Zero(receiver.Type().Elem()) + case mapError: + s.errorf("map has no entry for key %q", fieldName) + } + } + return result } } s.errorf("can't evaluate field %s in type %s", fieldName, typ) @@ -560,7 +587,15 @@ func (s *state) evalCall(dot, fun reflect.Value, node parse.Node, name string, a if final.IsValid() { t := typ.In(typ.NumIn() - 1) if typ.IsVariadic() { - t = t.Elem() + if numIn-1 < numFixed { + // The added final argument corresponds to a fixed parameter of the function. + // Validate against the type of the actual parameter. + t = typ.In(numIn - 1) + } else { + // The added final argument corresponds to the variadic part. + // Validate against the type of the elements of the variadic slice. + t = t.Elem() + } } argv[i] = s.validateType(final, t) } @@ -635,7 +670,7 @@ func (s *state) evalArg(dot reflect.Value, typ reflect.Type, n parse.Node) refle case *parse.PipeNode: return s.validateType(s.evalPipeline(dot, arg), typ) case *parse.IdentifierNode: - return s.evalFunction(dot, arg, arg, nil, zero) + return s.validateType(s.evalFunction(dot, arg, arg, nil, zero), typ) case *parse.ChainNode: return s.validateType(s.evalChainNode(dot, arg, nil, zero), typ) } diff --git a/libgo/go/text/template/exec_test.go b/libgo/go/text/template/exec_test.go index 69c213e..ba0e434 100644 --- a/libgo/go/text/template/exec_test.go +++ b/libgo/go/text/template/exec_test.go @@ -527,6 +527,24 @@ var execTests = []execTest{ {"bug12XE", "{{printf `%T` 0XEE}}", "int", T{}, true}, // Chained nodes did not work as arguments. Issue 8473. {"bug13", "{{print (.Copy).I}}", "17", tVal, true}, + // Didn't protect against nil or literal values in field chains. + {"bug14a", "{{(nil).True}}", "", tVal, false}, + {"bug14b", "{{$x := nil}}{{$x.anything}}", "", tVal, false}, + {"bug14c", `{{$x := (1.0)}}{{$y := ("hello")}}{{$x.anything}}{{$y.true}}`, "", tVal, false}, + // Didn't call validateType on function results. Issue 10800. + {"bug15", "{{valueString returnInt}}", "", tVal, false}, + // Variadic function corner cases. Issue 10946. + {"bug16a", "{{true|printf}}", "", tVal, false}, + {"bug16b", "{{1|printf}}", "", tVal, false}, + {"bug16c", "{{1.1|printf}}", "", tVal, false}, + {"bug16d", "{{'x'|printf}}", "", tVal, false}, + {"bug16e", "{{0i|printf}}", "", tVal, false}, + {"bug16f", "{{true|twoArgs \"xxx\"}}", "", tVal, false}, + {"bug16g", "{{\"aaa\" |twoArgs \"bbb\"}}", "twoArgs=bbbaaa", tVal, true}, + {"bug16h", "{{1|oneArg}}", "", tVal, false}, + {"bug16i", "{{\"aaa\"|oneArg}}", "oneArg=aaa", tVal, true}, + {"bug16j", "{{1+2i|printf \"%v\"}}", "(1+2i)", tVal, true}, + {"bug16k", "{{\"aaa\"|printf }}", "aaa", tVal, true}, } func zeroArgs() string { @@ -537,6 +555,10 @@ func oneArg(a string) string { return "oneArg=" + a } +func twoArgs(a, b string) string { + return "twoArgs=" + a + b +} + func dddArg(a int, b ...string) string { return fmt.Sprintln(a, b) } @@ -566,6 +588,11 @@ func valueString(v string) string { return "value is ignored" } +// returnInt returns an int +func returnInt() int { + return 7 +} + func add(args ...int) int { sum := 0 for _, x := range args { @@ -607,7 +634,9 @@ func testExecute(execTests []execTest, template *Template, t *testing.T) { "makemap": makemap, "mapOfThree": mapOfThree, "oneArg": oneArg, + "returnInt": returnInt, "stringer": stringer, + "twoArgs": twoArgs, "typeOf": typeOf, "valueString": valueString, "vfunc": vfunc, @@ -853,7 +882,13 @@ func TestTree(t *testing.T) { func TestExecuteOnNewTemplate(t *testing.T) { // This is issue 3872. - _ = New("Name").Templates() + New("Name").Templates() + // This is issue 11379. + new(Template).Templates() + new(Template).Parse("") + new(Template).New("abc").Parse("") + new(Template).Execute(nil, nil) // returns an error (but does not crash) + new(Template).ExecuteTemplate(nil, "XXX", nil) // returns an error (but does not crash) } const testTemplates = `{{define "one"}}one{{end}}{{define "two"}}two{{end}}` @@ -1042,3 +1077,67 @@ func TestComparison(t *testing.T) { } } } + +func TestMissingMapKey(t *testing.T) { + data := map[string]int{ + "x": 99, + } + tmpl, err := New("t1").Parse("{{.x}} {{.y}}") + if err != nil { + t.Fatal(err) + } + var b bytes.Buffer + // By default, just get "<no value>" + err = tmpl.Execute(&b, data) + if err != nil { + t.Fatal(err) + } + want := "99 <no value>" + got := b.String() + if got != want { + t.Errorf("got %q; expected %q", got, want) + } + // Same if we set the option explicitly to the default. + tmpl.Option("missingkey=default") + b.Reset() + err = tmpl.Execute(&b, data) + if err != nil { + t.Fatal("default:", err) + } + want = "99 <no value>" + got = b.String() + if got != want { + t.Errorf("got %q; expected %q", got, want) + } + // Next we ask for a zero value + tmpl.Option("missingkey=zero") + b.Reset() + err = tmpl.Execute(&b, data) + if err != nil { + t.Fatal("zero:", err) + } + want = "99 0" + got = b.String() + if got != want { + t.Errorf("got %q; expected %q", got, want) + } + // Now we ask for an error. + tmpl.Option("missingkey=error") + err = tmpl.Execute(&b, data) + if err == nil { + t.Errorf("expected error; got none") + } +} + +// Test that the error message for multiline unterminated string +// refers to the line number of the opening quote. +func TestUnterminatedStringError(t *testing.T) { + _, err := New("X").Parse("hello\n\n{{`unterminated\n\n\n\n}}\n some more\n\n") + if err == nil { + t.Fatal("expected error") + } + str := err.Error() + if !strings.Contains(str, "X:3: unexpected unterminated raw quoted strin") { + t.Fatalf("unexpected error: %s", str) + } +} diff --git a/libgo/go/text/template/funcs.go b/libgo/go/text/template/funcs.go index 39ee5ed..ccd0dfc 100644 --- a/libgo/go/text/template/funcs.go +++ b/libgo/go/text/template/funcs.go @@ -92,6 +92,8 @@ func goodFunc(typ reflect.Type) bool { // findFunction looks for a function in the template, and global map. func findFunction(name string, tmpl *Template) (reflect.Value, bool) { if tmpl != nil && tmpl.common != nil { + tmpl.muFuncs.RLock() + defer tmpl.muFuncs.RUnlock() if fn := tmpl.execFuncs[name]; fn.IsValid() { return fn, true } @@ -590,7 +592,7 @@ func evalArgs(args []interface{}) string { a, ok := printableValue(reflect.ValueOf(arg)) if ok { args[i] = a - } // else left fmt do its thing + } // else let fmt do its thing } s = fmt.Sprint(args...) } diff --git a/libgo/go/text/template/helper.go b/libgo/go/text/template/helper.go index 3636fb5..787ca62 100644 --- a/libgo/go/text/template/helper.go +++ b/libgo/go/text/template/helper.go @@ -26,8 +26,8 @@ func Must(t *Template, err error) *Template { } // ParseFiles creates a new Template and parses the template definitions from -// the named files. The returned template's name will have the (base) name and -// (parsed) contents of the first file. There must be at least one file. +// the named files. The returned template's name will have the base name and +// parsed contents of the first file. There must be at least one file. // If an error occurs, parsing stops and the returned *Template is nil. func ParseFiles(filenames ...string) (*Template, error) { return parseFiles(nil, filenames...) @@ -36,7 +36,13 @@ func ParseFiles(filenames ...string) (*Template, error) { // ParseFiles parses the named files and associates the resulting templates with // t. If an error occurs, parsing stops and the returned template is nil; // otherwise it is t. There must be at least one file. +// Since the templates created by ParseFiles are named by the base +// names of the argument files, t should usually have the name of one +// of the (base) names of the files. If it does not, depending on t's +// contents before calling ParseFiles, t.Execute may fail. In that +// case use t.ExecuteTemplate to execute a valid template. func (t *Template) ParseFiles(filenames ...string) (*Template, error) { + t.init() return parseFiles(t, filenames...) } @@ -92,6 +98,7 @@ func ParseGlob(pattern string) (*Template, error) { // equivalent to calling t.ParseFiles with the list of files matched by the // pattern. func (t *Template) ParseGlob(pattern string) (*Template, error) { + t.init() return parseGlob(t, pattern) } diff --git a/libgo/go/text/template/multi_test.go b/libgo/go/text/template/multi_test.go index e4e8048..ea01875 100644 --- a/libgo/go/text/template/multi_test.go +++ b/libgo/go/text/template/multi_test.go @@ -290,3 +290,76 @@ func TestRedefinition(t *testing.T) { t.Fatalf("expected redefinition error; got %v", err) } } + +// Issue 10879 +func TestEmptyTemplateCloneCrash(t *testing.T) { + t1 := New("base") + t1.Clone() // used to panic +} + +// Issue 10910, 10926 +func TestTemplateLookUp(t *testing.T) { + t1 := New("foo") + if t1.Lookup("foo") != nil { + t.Error("Lookup returned non-nil value for undefined template foo") + } + t1.New("bar") + if t1.Lookup("bar") != nil { + t.Error("Lookup returned non-nil value for undefined template bar") + } + t1.Parse(`{{define "foo"}}test{{end}}`) + if t1.Lookup("foo") == nil { + t.Error("Lookup returned nil value for defined template") + } +} + +func TestNew(t *testing.T) { + // template with same name already exists + t1, _ := New("test").Parse(`{{define "test"}}foo{{end}}`) + t2 := t1.New("test") + + if t1.common != t2.common { + t.Errorf("t1 & t2 didn't share common struct; got %v != %v", t1.common, t2.common) + } + if t1.Tree == nil { + t.Error("defined template got nil Tree") + } + if t2.Tree != nil { + t.Error("undefined template got non-nil Tree") + } + + containsT1 := false + for _, tmpl := range t1.Templates() { + if tmpl == t2 { + t.Error("Templates included undefined template") + } + if tmpl == t1 { + containsT1 = true + } + } + if !containsT1 { + t.Error("Templates didn't include defined template") + } +} + +func TestParse(t *testing.T) { + // In multiple calls to Parse with the same receiver template, only one call + // can contain text other than space, comments, and template definitions + var err error + t1 := New("test") + if _, err := t1.Parse(`{{define "test"}}{{end}}`); err != nil { + t.Fatalf("parsing test: %s", err) + } + if _, err := t1.Parse(`{{define "test"}}{{/* this is a comment */}}{{end}}`); err != nil { + t.Fatalf("parsing test: %s", err) + } + if _, err := t1.Parse(`{{define "test"}}foo{{end}}`); err != nil { + t.Fatalf("parsing test: %s", err) + } + if _, err = t1.Parse(`{{define "test"}}foo{{end}}`); err == nil { + t.Fatal("no error from redefining a template") + } + if !strings.Contains(err.Error(), "redefinition") { + t.Fatalf("expected redefinition error; got %v", err) + } +} diff --git a/libgo/go/text/template/option.go b/libgo/go/text/template/option.go new file mode 100644 index 0000000..addce2d --- /dev/null +++ b/libgo/go/text/template/option.go @@ -0,0 +1,74 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the code to handle template options. + +package template + +import "strings" + +// missingKeyAction defines how to respond to indexing a map with a key that is not present. +type missingKeyAction int + +const ( + mapInvalid missingKeyAction = iota // Return an invalid reflect.Value. + mapZeroValue // Return the zero value for the map element. + mapError // Error out +) + +type option struct { + missingKey missingKeyAction +} + +// Option sets options for the template. Options are described by +// strings, either a simple string or "key=value". There can be at +// most one equals sign in an option string. If the option string +// is unrecognized or otherwise invalid, Option panics. +// +// Known options: +// +// missingkey: Control the behavior during execution if a map is +// indexed with a key that is not present in the map. +// "missingkey=default" or "missingkey=invalid" +// The default behavior: Do nothing and continue execution. +// If printed, the result of the index operation is the string +// "<no value>". +// "missingkey=zero" +// The operation returns the zero value for the map type's element. +// "missingkey=error" +// Execution stops immediately with an error. +// +func (t *Template) Option(opt ...string) *Template { + t.init() + for _, s := range opt { + t.setOption(s) + } + return t +} + +func (t *Template) setOption(opt string) { + if opt == "" { + panic("empty option string") + } + elems := strings.Split(opt, "=") + switch len(elems) { + case 2: + // key=value + switch elems[0] { + case "missingkey": + switch elems[1] { + case "invalid", "default": + t.option.missingKey = mapInvalid + return + case "zero": + t.option.missingKey = mapZeroValue + return + case "error": + t.option.missingKey = mapError + return + } + } + } + panic("unrecognized option: " + opt) +} diff --git a/libgo/go/text/template/parse/lex.go b/libgo/go/text/template/parse/lex.go index 1674aaf..8f9fe1d 100644 --- a/libgo/go/text/template/parse/lex.go +++ b/libgo/go/text/template/parse/lex.go @@ -167,12 +167,20 @@ func (l *lexer) errorf(format string, args ...interface{}) stateFn { } // nextItem returns the next item from the input. +// Called by the parser, not in the lexing goroutine. func (l *lexer) nextItem() item { item := <-l.items l.lastPos = item.pos return item } +// drain drains the output so the lexing goroutine will exit. +// Called by the parser, not in the lexing goroutine. +func (l *lexer) drain() { + for range l.items { + } +} + // lex creates a new scanner for the input string. func lex(name, input, left, right string) *lexer { if left == "" { @@ -197,6 +205,7 @@ func (l *lexer) run() { for l.state = lexText; l.state != nil; { l.state = l.state(l) } + close(l.items) } // state functions @@ -313,14 +322,12 @@ func lexInsideAction(l *lexer) stateFn { case r == '(': l.emit(itemLeftParen) l.parenDepth++ - return lexInsideAction case r == ')': l.emit(itemRightParen) l.parenDepth-- if l.parenDepth < 0 { return l.errorf("unexpected right paren %#U", r) } - return lexInsideAction case r <= unicode.MaxASCII && unicode.IsPrint(r): l.emit(itemChar) return lexInsideAction @@ -525,7 +532,7 @@ func lexRawQuote(l *lexer) stateFn { Loop: for { switch l.next() { - case eof, '\n': + case eof: return l.errorf("unterminated raw quoted string") case '`': break Loop diff --git a/libgo/go/text/template/parse/lex_test.go b/libgo/go/text/template/parse/lex_test.go index d251ccf..be551d8 100644 --- a/libgo/go/text/template/parse/lex_test.go +++ b/libgo/go/text/template/parse/lex_test.go @@ -58,18 +58,20 @@ type lexTest struct { } var ( - tEOF = item{itemEOF, 0, ""} - tFor = item{itemIdentifier, 0, "for"} - tLeft = item{itemLeftDelim, 0, "{{"} - tLpar = item{itemLeftParen, 0, "("} - tPipe = item{itemPipe, 0, "|"} - tQuote = item{itemString, 0, `"abc \n\t\" "`} - tRange = item{itemRange, 0, "range"} - tRight = item{itemRightDelim, 0, "}}"} - tRpar = item{itemRightParen, 0, ")"} - tSpace = item{itemSpace, 0, " "} - raw = "`" + `abc\n\t\" ` + "`" - tRawQuote = item{itemRawString, 0, raw} + tEOF = item{itemEOF, 0, ""} + tFor = item{itemIdentifier, 0, "for"} + tLeft = item{itemLeftDelim, 0, "{{"} + tLpar = item{itemLeftParen, 0, "("} + tPipe = item{itemPipe, 0, "|"} + tQuote = item{itemString, 0, `"abc \n\t\" "`} + tRange = item{itemRange, 0, "range"} + tRight = item{itemRightDelim, 0, "}}"} + tRpar = item{itemRightParen, 0, ")"} + tSpace = item{itemSpace, 0, " "} + raw = "`" + `abc\n\t\" ` + "`" + rawNL = "`now is{{\n}}the time`" // Contains newline inside raw quote. + tRawQuote = item{itemRawString, 0, raw} + tRawQuoteNL = item{itemRawString, 0, rawNL} ) var lexTests = []lexTest{ @@ -104,6 +106,7 @@ var lexTests = []lexTest{ {"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}}, {"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}}, {"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}}, + {"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}}, {"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{ tLeft, {itemNumber, 0, "1"}, @@ -294,7 +297,7 @@ var lexTests = []lexTest{ tLeft, {itemError, 0, "unterminated quoted string"}, }}, - {"unclosed raw quote", "{{`xx\n`}}", []item{ + {"unclosed raw quote", "{{`xx}}", []item{ tLeft, {itemError, 0, "unterminated raw quoted string"}, }}, @@ -463,3 +466,31 @@ func TestPos(t *testing.T) { } } } + +// Test that an error shuts down the lexing goroutine. +func TestShutdown(t *testing.T) { + // We need to duplicate template.Parse here to hold on to the lexer. + const text = "erroneous{{define}}{{else}}1234" + lexer := lex("foo", text, "{{", "}}") + _, err := New("root").parseLexer(lexer, text) + if err == nil { + t.Fatalf("expected error") + } + // The error should have drained the input. Therefore, the lexer should be shut down. + token, ok := <-lexer.items + if ok { + t.Fatalf("input was not drained; got %v", token) + } +} + +// parseLexer is a local version of parse that lets us pass in the lexer instead of building it. +// We expect an error, so the tree set and funcs list are explicitly nil. +func (t *Tree) parseLexer(lex *lexer, text string) (tree *Tree, err error) { + defer t.recover(&err) + t.ParseName = t.Name + t.startParse(nil, lex) + t.parse(nil) + t.add(nil) + t.stopParse() + return t, nil +} diff --git a/libgo/go/text/template/parse/node.go b/libgo/go/text/template/parse/node.go index 55c37f6..55ff46c 100644 --- a/libgo/go/text/template/parse/node.go +++ b/libgo/go/text/template/parse/node.go @@ -145,7 +145,7 @@ type PipeNode struct { NodeType Pos tr *Tree - Line int // The line number in the input (deprecated; kept for compatibility) + Line int // The line number in the input. Deprecated: Kept for compatibility. Decl []*VariableNode // Variable declarations in lexical order. Cmds []*CommandNode // The commands in lexical order. } @@ -208,7 +208,7 @@ type ActionNode struct { NodeType Pos tr *Tree - Line int // The line number in the input (deprecated; kept for compatibility) + Line int // The line number in the input. Deprecated: Kept for compatibility. Pipe *PipeNode // The pipeline in the action. } @@ -592,6 +592,11 @@ func (t *Tree) newNumber(pos Pos, text string, typ itemType) (*NumberNode, error } else { f, err := strconv.ParseFloat(text, 64) if err == nil { + // If we parsed it as a float but it looks like an integer, + // it's a huge number too large to fit in an int. Reject it. + if !strings.ContainsAny(text, ".eE") { + return nil, fmt.Errorf("integer overflow: %q", text) + } n.IsFloat = true n.Float64 = f // If a floating-point extraction succeeded, extract the int if needed. @@ -696,7 +701,7 @@ type elseNode struct { NodeType Pos tr *Tree - Line int // The line number in the input (deprecated; kept for compatibility) + Line int // The line number in the input. Deprecated: Kept for compatibility. } func (t *Tree) newElse(pos Pos, line int) *elseNode { @@ -724,7 +729,7 @@ type BranchNode struct { NodeType Pos tr *Tree - Line int // The line number in the input (deprecated; kept for compatibility) + Line int // The line number in the input. Deprecated: Kept for compatibility. Pipe *PipeNode // The pipeline to be evaluated. List *ListNode // What to execute if the value is non-empty. ElseList *ListNode // What to execute if the value is empty (nil if absent). @@ -809,7 +814,7 @@ type TemplateNode struct { NodeType Pos tr *Tree - Line int // The line number in the input (deprecated; kept for compatibility) + Line int // The line number in the input. Deprecated: Kept for compatibility. Name string // The name of the template (unquoted). Pipe *PipeNode // The command to evaluate as dot for the template. } diff --git a/libgo/go/text/template/parse/parse.go b/libgo/go/text/template/parse/parse.go index af33880..88aacd1 100644 --- a/libgo/go/text/template/parse/parse.go +++ b/libgo/go/text/template/parse/parse.go @@ -196,6 +196,7 @@ func (t *Tree) recover(errp *error) { panic(e) } if t != nil { + t.lex.drain() t.stopParse() } *errp = e.(error) @@ -288,11 +289,12 @@ func (t *Tree) parse(treeSet map[string]*Tree) (next Node) { } t.backup2(delim) } - n := t.textOrAction() - if n.Type() == nodeEnd { + switch n := t.textOrAction(); n.Type() { + case nodeEnd, nodeElse: t.errorf("unexpected %s", n) + default: + t.Root.append(n) } - t.Root.append(n) } return nil } @@ -411,9 +413,8 @@ func (t *Tree) pipeline(context string) (pipe *PipeNode) { for { switch token := t.nextNonSpace(); token.typ { case itemRightDelim, itemRightParen: - if len(pipe.Cmds) == 0 { - t.errorf("missing value for %s", context) - } + // At this point, the pipeline is complete + t.checkPipeline(pipe, context) if token.typ == itemRightParen { t.backup() } @@ -428,6 +429,21 @@ func (t *Tree) pipeline(context string) (pipe *PipeNode) { } } +func (t *Tree) checkPipeline(pipe *PipeNode, context string) { + // Reject empty pipelines + if len(pipe.Cmds) == 0 { + t.errorf("missing value for %s", context) + } + // Only the first command of a pipeline can start with a non executable operand + for i, c := range pipe.Cmds[1:] { + switch c.Args[0].Type() { + case NodeBool, NodeDot, NodeNil, NodeNumber, NodeString: + // With A|B|C, pipeline stage 2 is B + t.errorf("non executable command in pipeline stage %d", i+2) + } + } +} + func (t *Tree) parseControl(allowElseIf bool, context string) (pos Pos, line int, pipe *PipeNode, list, elseList *ListNode) { defer t.popVars(len(t.vars)) line = t.lex.lineNumber() @@ -553,7 +569,7 @@ func (t *Tree) command() *CommandNode { t.backup() case itemPipe: default: - t.errorf("unexpected %s in operand; missing space?", token) + t.errorf("unexpected %s in operand", token) } break } @@ -581,12 +597,15 @@ func (t *Tree) operand() Node { // Compatibility with original API: If the term is of type NodeField // or NodeVariable, just put more fields on the original. // Otherwise, keep the Chain node. - // TODO: Switch to Chains always when we can. + // Obvious parsing errors involving literal values are detected here. + // More complex error cases will have to be handled at execution time. switch node.Type() { case NodeField: node = t.newField(chain.Position(), chain.String()) case NodeVariable: node = t.newVariable(chain.Position(), chain.String()) + case NodeBool, NodeString, NodeNumber, NodeNil, NodeDot: + t.errorf("unexpected . after term %q", node.String()) default: node = chain } diff --git a/libgo/go/text/template/parse/parse_test.go b/libgo/go/text/template/parse/parse_test.go index 4a504fa..200d50c 100644 --- a/libgo/go/text/template/parse/parse_test.go +++ b/libgo/go/text/template/parse/parse_test.go @@ -69,6 +69,7 @@ var numberTests = []numberTest{ {text: "1+2."}, {text: "'x"}, {text: "'xx'"}, + {text: "'433937734937734969526500969526500'"}, // Integer too large - issue 10634. // Issue 8622 - 0xe parsed as floating point. Very embarrassing. {"0xef", true, true, true, false, 0xef, 0xef, 0xef, 0}, } @@ -230,6 +231,9 @@ var parseTests = []parseTest{ // Errors. {"unclosed action", "hello{{range", hasError, ""}, {"unmatched end", "{{end}}", hasError, ""}, + {"unmatched else", "{{else}}", hasError, ""}, + {"unmatched else after if", "{{if .X}}hello{{end}}{{else}}", hasError, ""}, + {"multiple else", "{{if .X}}1{{else}}2{{else}}3{{end}}", hasError, ""}, {"missing end", "hello{{range .x}}", hasError, ""}, {"missing end after else", "hello{{range .x}}{{else}}", hasError, ""}, {"undefined function", "hello{{undefined}}", hasError, ""}, @@ -257,6 +261,22 @@ var parseTests = []parseTest{ {"bug1a", "{{$x:=.}}{{$x!2}}", hasError, ""}, // ! is just illegal here. {"bug1b", "{{$x:=.}}{{$x+2}}", hasError, ""}, // $x+2 should not parse as ($x) (+2). {"bug1c", "{{$x:=.}}{{$x +2}}", noError, "{{$x := .}}{{$x +2}}"}, // It's OK with a space. + // dot following a literal value + {"dot after integer", "{{1.E}}", hasError, ""}, + {"dot after float", "{{0.1.E}}", hasError, ""}, + {"dot after boolean", "{{true.E}}", hasError, ""}, + {"dot after char", "{{'a'.any}}", hasError, ""}, + {"dot after string", `{{"hello".guys}}`, hasError, ""}, + {"dot after dot", "{{..E}}", hasError, ""}, + {"dot after nil", "{{nil.E}}", hasError, ""}, + // Wrong pipeline + {"wrong pipeline dot", "{{12|.}}", hasError, ""}, + {"wrong pipeline number", "{{.|12|printf}}", hasError, ""}, + {"wrong pipeline string", "{{.|printf|\"error\"}}", hasError, ""}, + {"wrong pipeline char", "{{12|printf|'e'}}", hasError, ""}, + {"wrong pipeline boolean", "{{.|true}}", hasError, ""}, + {"wrong pipeline nil", "{{'c'|nil}}", hasError, ""}, + {"empty pipeline", `{{printf "%d" ( ) }}`, hasError, ""}, } var builtins = map[string]interface{}{ @@ -375,7 +395,7 @@ var errorTests = []parseTest{ hasError, `unexpected ")"`}, {"space", "{{`x`3}}", - hasError, `missing space?`}, + hasError, `in operand`}, {"idchar", "{{a#}}", hasError, `'#'`}, @@ -407,6 +427,15 @@ var errorTests = []parseTest{ {"undefvar", "{{$a}}", hasError, `undefined variable`}, + {"wrongdot", + "{{true.any}}", + hasError, `unexpected . after term`}, + {"wrongpipeline", + "{{12|false}}", + hasError, `non executable command in pipeline`}, + {"emptypipeline", + `{{ ( ) }}`, + hasError, `missing value for parenthesized pipeline`}, } func TestErrors(t *testing.T) { diff --git a/libgo/go/text/template/template.go b/libgo/go/text/template/template.go index 249d0cb..3e80982 100644 --- a/libgo/go/text/template/template.go +++ b/libgo/go/text/template/template.go @@ -7,15 +7,18 @@ package template import ( "fmt" "reflect" + "sync" "text/template/parse" ) // common holds the information shared by related templates. type common struct { - tmpl map[string]*Template + tmpl map[string]*Template // Map from name to defined templates. + option option // We use two maps, one for parsing and one for execution. // This separation makes the API cleaner since it doesn't // expose reflection to the client. + muFuncs sync.RWMutex // protects parseFuncs and execFuncs parseFuncs FuncMap execFuncs map[string]reflect.Value } @@ -31,11 +34,13 @@ type Template struct { rightDelim string } -// New allocates a new template with the given name. +// New allocates a new, undefined template with the given name. func New(name string) *Template { - return &Template{ + t := &Template{ name: name, } + t.init() + return t } // Name returns the name of the template. @@ -43,25 +48,28 @@ func (t *Template) Name() string { return t.name } -// New allocates a new template associated with the given one and with the same +// New allocates a new, undefined template associated with the given one and with the same // delimiters. The association, which is transitive, allows one template to // invoke another with a {{template}} action. func (t *Template) New(name string) *Template { t.init() - return &Template{ + nt := &Template{ name: name, common: t.common, leftDelim: t.leftDelim, rightDelim: t.rightDelim, } + return nt } +// init guarantees that t has a valid common structure. func (t *Template) init() { if t.common == nil { - t.common = new(common) - t.tmpl = make(map[string]*Template) - t.parseFuncs = make(FuncMap) - t.execFuncs = make(map[string]reflect.Value) + c := new(common) + c.tmpl = make(map[string]*Template) + c.parseFuncs = make(FuncMap) + c.execFuncs = make(map[string]reflect.Value) + t.common = c } } @@ -74,15 +82,20 @@ func (t *Template) init() { func (t *Template) Clone() (*Template, error) { nt := t.copy(nil) nt.init() - nt.tmpl[t.name] = nt + if t.common == nil { + return nt, nil + } for k, v := range t.tmpl { - if k == t.name { // Already installed. + if k == t.name { + nt.tmpl[t.name] = nt continue } // The associated templates share nt's common structure. tmpl := v.copy(nt.common) nt.tmpl[k] = tmpl } + t.muFuncs.RLock() + defer t.muFuncs.RUnlock() for k, v := range t.parseFuncs { nt.parseFuncs[k] = v } @@ -102,20 +115,27 @@ func (t *Template) copy(c *common) *Template { return nt } -// AddParseTree creates a new template with the name and parse tree -// and associates it with t. +// AddParseTree adds parse tree for template with given name and associates it with t. +// If the template does not already exist, it will create a new one. +// It is an error to reuse a name except to overwrite an empty template. func (t *Template) AddParseTree(name string, tree *parse.Tree) (*Template, error) { - if t.common != nil && t.tmpl[name] != nil { - return nil, fmt.Errorf("template: redefinition of template %q", name) + t.init() + // If the name is the name of this template, overwrite this template. + // The associate method checks it's not a redefinition. + nt := t + if name != t.name { + nt = t.New(name) + } + // Even if nt == t, we need to install it in the common.tmpl map. + if replace, err := t.associate(nt, tree); err != nil { + return nil, err + } else if replace { + nt.Tree = tree } - nt := t.New(name) - nt.Tree = tree - t.tmpl[name] = nt return nt, nil } -// Templates returns a slice of the templates associated with t, including t -// itself. +// Templates returns a slice of defined templates associated with t. func (t *Template) Templates() []*Template { if t.common == nil { return nil @@ -134,6 +154,7 @@ func (t *Template) Templates() []*Template { // corresponding default: {{ or }}. // The return value is the template, so calls can be chained. func (t *Template) Delims(left, right string) *Template { + t.init() t.leftDelim = left t.rightDelim = right return t @@ -145,13 +166,15 @@ func (t *Template) Delims(left, right string) *Template { // value is the template, so calls can be chained. func (t *Template) Funcs(funcMap FuncMap) *Template { t.init() + t.muFuncs.Lock() + defer t.muFuncs.Unlock() addValueFuncs(t.execFuncs, funcMap) addFuncs(t.parseFuncs, funcMap) return t } -// Lookup returns the template with the given name that is associated with t, -// or nil if there is no such template. +// Lookup returns the template with the given name that is associated with t. +// It returns nil if there is no such template or the template has no definition. func (t *Template) Lookup(name string) *Template { if t.common == nil { return nil @@ -159,7 +182,7 @@ func (t *Template) Lookup(name string) *Template { return t.tmpl[name] } -// Parse parses a string into a template. Nested template definitions will be +// Parse defines the template by parsing the text. Nested template definitions will be // associated with the top-level template t. Parse may be called multiple times // to parse definitions of templates to associate with t. It is an error if a // resulting template is non-empty (contains content other than template @@ -168,26 +191,17 @@ func (t *Template) Lookup(name string) *Template { // can contain text other than space, comments, and template definitions.) func (t *Template) Parse(text string) (*Template, error) { t.init() + t.muFuncs.RLock() trees, err := parse.Parse(t.name, text, t.leftDelim, t.rightDelim, t.parseFuncs, builtins) + t.muFuncs.RUnlock() if err != nil { return nil, err } // Add the newly parsed trees, including the one for t, into our common structure. for name, tree := range trees { - // If the name we parsed is the name of this template, overwrite this template. - // The associate method checks it's not a redefinition. - tmpl := t - if name != t.name { - tmpl = t.New(name) - } - // Even if t == tmpl, we need to install it in the common.tmpl map. - if replace, err := t.associate(tmpl, tree); err != nil { + if _, err := t.AddParseTree(name, tree); err != nil { return nil, err - } else if replace { - tmpl.Tree = tree } - tmpl.leftDelim = t.leftDelim - tmpl.rightDelim = t.rightDelim } return t, nil } |