diff options
Diffstat (limited to 'libgo/go/html')
-rw-r--r-- | libgo/go/html/doc.go | 3 | ||||
-rw-r--r-- | libgo/go/html/parse.go | 353 | ||||
-rw-r--r-- | libgo/go/html/parse_test.go | 22 | ||||
-rw-r--r-- | libgo/go/html/render.go | 12 | ||||
-rw-r--r-- | libgo/go/html/token.go | 50 | ||||
-rw-r--r-- | libgo/go/html/token_test.go | 1 |
6 files changed, 368 insertions, 73 deletions
diff --git a/libgo/go/html/doc.go b/libgo/go/html/doc.go index 5bc0630..ba9d188 100644 --- a/libgo/go/html/doc.go +++ b/libgo/go/html/doc.go @@ -70,9 +70,6 @@ call to Next. For example, to extract an HTML page's anchor text: } } -A Tokenizer typically skips over HTML comments. To return comment tokens, set -Tokenizer.ReturnComments to true before looping over calls to Next. - Parsing is done by calling Parse with an io.Reader, which returns the root of the parse tree (the document element) as a *Node. It is the caller's responsibility to ensure that the Reader provides UTF-8 encoded HTML. For diff --git a/libgo/go/html/parse.go b/libgo/go/html/parse.go index 582437f..530942aa 100644 --- a/libgo/go/html/parse.go +++ b/libgo/go/html/parse.go @@ -32,6 +32,9 @@ type parser struct { // originalIM is the insertion mode to go back to after completing a text // or inTableText insertion mode. originalIM insertionMode + // fosterParenting is whether new elements should be inserted according to + // the foster parenting rules (section 11.2.5.3). + fosterParenting bool } func (p *parser) top() *Node { @@ -49,6 +52,11 @@ var ( tableScopeStopTags = []string{"html", "table"} ) +// stopTags for use in clearStackToContext. +var ( + tableRowContextStopTags = []string{"tr", "html"} +) + // popUntil pops the stack of open elements at the highest element whose tag // is in matchTags, provided there is no higher element in stopTags. It returns // whether or not there was such an element. If there was not, popUntil leaves @@ -103,12 +111,61 @@ func (p *parser) elementInScope(stopTags []string, matchTags ...string) bool { // addChild adds a child node n to the top element, and pushes n onto the stack // of open elements if it is an element node. func (p *parser) addChild(n *Node) { - p.top().Add(n) + if p.fosterParenting { + p.fosterParent(n) + } else { + p.top().Add(n) + } + if n.Type == ElementNode { p.oe = append(p.oe, n) } } +// fosterParent adds a child node according to the foster parenting rules. +// Section 11.2.5.3, "foster parenting". +func (p *parser) fosterParent(n *Node) { + var table, parent *Node + var i int + for i = len(p.oe) - 1; i >= 0; i-- { + if p.oe[i].Data == "table" { + table = p.oe[i] + break + } + } + + if table == nil { + // The foster parent is the html element. + parent = p.oe[0] + } else { + parent = table.Parent + } + if parent == nil { + parent = p.oe[i-1] + } + + var child *Node + for i, child = range parent.Child { + if child == table { + break + } + } + + if i > 0 && parent.Child[i-1].Type == TextNode && n.Type == TextNode { + parent.Child[i-1].Data += n.Data + return + } + + if i == len(parent.Child) { + parent.Add(n) + } else { + // Insert n into parent.Child at index i. + parent.Child = append(parent.Child[:i+1], parent.Child[i:]...) + parent.Child[i] = n + n.Parent = parent + } +} + // addText adds text to the preceding node if it is a text node, or else it // calls addChild with a new text node. func (p *parser) addText(text string) { @@ -170,9 +227,9 @@ func (p *parser) reconstructActiveFormattingElements() { } for { i++ - n = p.afe[i] - p.addChild(n.clone()) - p.afe[i] = n + clone := p.afe[i].clone() + p.addChild(clone) + p.afe[i] = clone if i == len(p.afe)-1 { break } @@ -234,10 +291,52 @@ func (p *parser) setOriginalIM(im insertionMode) { p.originalIM = im } +// Section 11.2.3.1, "reset the insertion mode". +func (p *parser) resetInsertionMode() insertionMode { + for i := len(p.oe) - 1; i >= 0; i-- { + n := p.oe[i] + if i == 0 { + // TODO: set n to the context element, for HTML fragment parsing. + } + switch n.Data { + case "select": + return inSelectIM + case "td", "th": + return inCellIM + case "tr": + return inRowIM + case "tbody", "thead", "tfoot": + return inTableBodyIM + case "caption": + // TODO: return inCaptionIM + case "colgroup": + // TODO: return inColumnGroupIM + case "table": + return inTableIM + case "head": + return inBodyIM + case "body": + return inBodyIM + case "frameset": + // TODO: return inFramesetIM + case "html": + return beforeHeadIM + } + } + return inBodyIM +} + // Section 11.2.5.4.1. func initialIM(p *parser) (insertionMode, bool) { - if p.tok.Type == DoctypeToken { - p.addChild(&Node{ + switch p.tok.Type { + case CommentToken: + p.doc.Add(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return initialIM, true + case DoctypeToken: + p.doc.Add(&Node{ Type: DoctypeNode, Data: p.tok.Data, }) @@ -275,6 +374,12 @@ func beforeHTMLIM(p *parser) (insertionMode, bool) { default: // Ignore the token. } + case CommentToken: + p.doc.Add(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return beforeHTMLIM, true } if add || implied { p.addElement("html", attr) @@ -312,6 +417,12 @@ func beforeHeadIM(p *parser) (insertionMode, bool) { default: // Ignore the token. } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return beforeHeadIM, true } if add || implied { p.addElement("head", attr) @@ -344,11 +455,17 @@ func inHeadIM(p *parser) (insertionMode, bool) { pop = true } // TODO. + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return inHeadIM, true } if pop || implied { n := p.oe.pop() if n.Data != "head" { - panic("html: bad parser state") + panic("html: bad parser state: <head> element not found, in the in-head insertion mode") } return afterHeadIM, !implied } @@ -387,6 +504,12 @@ func afterHeadIM(p *parser) (insertionMode, bool) { } case EndTagToken: // TODO. + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return afterHeadIM, true } if add || implied { p.addElement("body", attr) @@ -447,6 +570,30 @@ func inBodyIM(p *parser) (insertionMode, bool) { p.oe.pop() p.acknowledgeSelfClosingTag() p.framesetOK = false + case "select": + p.reconstructActiveFormattingElements() + p.addElement(p.tok.Data, p.tok.Attr) + p.framesetOK = false + // TODO: detect <select> inside a table. + return inSelectIM, true + case "li": + p.framesetOK = false + for i := len(p.oe) - 1; i >= 0; i-- { + node := p.oe[i] + switch node.Data { + case "li": + p.popUntil(listItemScopeStopTags, "li") + case "address", "div", "p": + continue + default: + if !isSpecialElement[node.Data] { + continue + } + } + break + } + p.popUntil(buttonScopeStopTags, "p") + p.addElement("li", p.tok.Attr) default: // TODO. p.addElement(p.tok.Data, p.tok.Attr) @@ -463,12 +610,16 @@ func inBodyIM(p *parser) (insertionMode, bool) { p.popUntil(buttonScopeStopTags, "p") case "a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u": p.inBodyEndTagFormatting(p.tok.Data) + case "address", "article", "aside", "blockquote", "button", "center", "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", "footer", "header", "hgroup", "listing", "menu", "nav", "ol", "pre", "section", "summary", "ul": + p.popUntil(defaultScopeStopTags, p.tok.Data) default: - // TODO: any other end tag - if p.tok.Data == p.top().Data { - p.oe.pop() - } + p.inBodyEndTagOther(p.tok.Data) } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) } return inBodyIM, true @@ -496,6 +647,7 @@ func (p *parser) inBodyEndTagFormatting(tag string) { } } if formattingElement == nil { + p.inBodyEndTagOther(tag) return } feIndex := p.oe.index(formattingElement) @@ -568,8 +720,7 @@ func (p *parser) inBodyEndTagFormatting(tag string) { } switch commonAncestor.Data { case "table", "tbody", "tfoot", "thead", "tr": - // TODO: fix up misnested table nodes; find the foster parent. - fallthrough + p.fosterParent(lastNode) default: commonAncestor.Add(lastNode) } @@ -590,6 +741,19 @@ func (p *parser) inBodyEndTagFormatting(tag string) { } } +// inBodyEndTagOther performs the "any other end tag" algorithm for inBodyIM. +func (p *parser) inBodyEndTagOther(tag string) { + for i := len(p.oe) - 1; i >= 0; i-- { + if p.oe[i].Data == tag { + p.oe = p.oe[:i] + break + } + if isSpecialElement[p.oe[i].Data] { + break + } + } +} + // Section 11.2.5.4.8. func textIM(p *parser) (insertionMode, bool) { switch p.tok.Type { @@ -606,12 +770,6 @@ func textIM(p *parser) (insertionMode, bool) { // Section 11.2.5.4.9. func inTableIM(p *parser) (insertionMode, bool) { - var ( - add bool - data string - attr []Attribute - consumed bool - ) switch p.tok.Type { case ErrorToken: // Stop parsing. @@ -621,13 +779,19 @@ func inTableIM(p *parser) (insertionMode, bool) { case StartTagToken: switch p.tok.Data { case "tbody", "tfoot", "thead": - add = true - data = p.tok.Data - attr = p.tok.Attr - consumed = true + p.clearStackToContext(tableScopeStopTags) + p.addElement(p.tok.Data, p.tok.Attr) + return inTableBodyIM, true case "td", "th", "tr": - add = true - data = "tbody" + p.clearStackToContext(tableScopeStopTags) + p.addElement("tbody", nil) + return inTableBodyIM, false + case "table": + if p.popUntil(tableScopeStopTags, "table") { + return p.resetInsertionMode(), false + } + // Ignore the token. + return inTableIM, true default: // TODO. } @@ -635,8 +799,7 @@ func inTableIM(p *parser) (insertionMode, bool) { switch p.tok.Data { case "table": if p.popUntil(tableScopeStopTags, "table") { - // TODO: "reset the insertion mode appropriately" as per 11.2.3.1. - return inBodyIM, false + return p.resetInsertionMode(), true } // Ignore the token. return inTableIM, true @@ -644,14 +807,34 @@ func inTableIM(p *parser) (insertionMode, bool) { // Ignore the token. return inTableIM, true } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return inTableIM, true } - if add { - // TODO: clear the stack back to a table context. - p.addElement(data, attr) - return inTableBodyIM, consumed + + switch p.top().Data { + case "table", "tbody", "tfoot", "thead", "tr": + p.fosterParenting = true + defer func() { p.fosterParenting = false }() + } + + return useTheRulesFor(p, inTableIM, inBodyIM) +} + +// clearStackToContext pops elements off the stack of open elements +// until an element listed in stopTags is found. +func (p *parser) clearStackToContext(stopTags []string) { + for i := len(p.oe) - 1; i >= 0; i-- { + for _, tag := range stopTags { + if p.oe[i].Data == tag { + p.oe = p.oe[:i+1] + return + } + } } - // TODO: return useTheRulesFor(inTableIM, inBodyIM, p) unless etc. etc. foster parenting. - return inTableIM, true } // Section 11.2.5.4.13. @@ -693,6 +876,12 @@ func inTableBodyIM(p *parser) (insertionMode, bool) { // Ignore the token. return inTableBodyIM, true } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return inTableBodyIM, true } if add { // TODO: clear the stack back to a table body context. @@ -722,7 +911,12 @@ func inRowIM(p *parser) (insertionMode, bool) { case EndTagToken: switch p.tok.Data { case "tr": - // TODO. + if !p.elementInScope(tableScopeStopTags, "tr") { + return inRowIM, true + } + p.clearStackToContext(tableRowContextStopTags) + p.oe.pop() + return inTableBodyIM, true case "table": if p.popUntil(tableScopeStopTags, "tr") { return inTableBodyIM, false @@ -737,6 +931,12 @@ func inRowIM(p *parser) (insertionMode, bool) { default: // TODO. } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return inRowIM, true } return useTheRulesFor(p, inRowIM, inTableIM) } @@ -763,6 +963,12 @@ func inCellIM(p *parser) (insertionMode, bool) { // TODO: check for matching element in table scope. closeTheCellAndReprocess = true } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return inCellIM, true } if closeTheCellAndReprocess { if p.popUntil(tableScopeStopTags, "td") || p.popUntil(tableScopeStopTags, "th") { @@ -773,6 +979,68 @@ func inCellIM(p *parser) (insertionMode, bool) { return useTheRulesFor(p, inCellIM, inBodyIM) } +// Section 11.2.5.4.16. +func inSelectIM(p *parser) (insertionMode, bool) { + endSelect := false + switch p.tok.Type { + case ErrorToken: + // TODO. + case TextToken: + p.addText(p.tok.Data) + case StartTagToken: + switch p.tok.Data { + case "html": + // TODO. + case "option": + if p.top().Data == "option" { + p.oe.pop() + } + p.addElement(p.tok.Data, p.tok.Attr) + case "optgroup": + // TODO. + case "select": + endSelect = true + case "input", "keygen", "textarea": + // TODO. + case "script": + // TODO. + default: + // Ignore the token. + } + case EndTagToken: + switch p.tok.Data { + case "option": + // TODO. + case "optgroup": + // TODO. + case "select": + endSelect = true + default: + // Ignore the token. + } + case CommentToken: + p.doc.Add(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + } + if endSelect { + for i := len(p.oe) - 1; i >= 0; i-- { + switch p.oe[i].Data { + case "select": + p.oe = p.oe[:i] + return p.resetInsertionMode(), true + case "option", "optgroup": + continue + default: + // Ignore the token. + return inSelectIM, true + } + } + } + return inSelectIM, true +} + // Section 11.2.5.4.18. func afterBodyIM(p *parser) (insertionMode, bool) { switch p.tok.Type { @@ -790,7 +1058,18 @@ func afterBodyIM(p *parser) (insertionMode, bool) { default: // TODO. } + case CommentToken: + // The comment is attached to the <html> element. + if len(p.oe) < 1 || p.oe[0].Data != "html" { + panic("html: bad parser state: <html> element not found, in the after-body insertion mode") + } + p.oe[0].Add(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return afterBodyIM, true } + // TODO: should this be "return inBodyIM, true"? return afterBodyIM, true } @@ -806,6 +1085,12 @@ func afterAfterBodyIM(p *parser) (insertionMode, bool) { if p.tok.Data == "html" { return useTheRulesFor(p, afterAfterBodyIM, inBodyIM) } + case CommentToken: + p.doc.Add(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + return afterAfterBodyIM, true } return inBodyIM, false } diff --git a/libgo/go/html/parse_test.go b/libgo/go/html/parse_test.go index 564580c..b0ddd92 100644 --- a/libgo/go/html/parse_test.go +++ b/libgo/go/html/parse_test.go @@ -69,11 +69,15 @@ func readDat(filename string, c chan io.Reader) { } } -func dumpLevel(w io.Writer, n *Node, level int) os.Error { +func dumpIndent(w io.Writer, level int) { io.WriteString(w, "| ") for i := 0; i < level; i++ { io.WriteString(w, " ") } +} + +func dumpLevel(w io.Writer, n *Node, level int) os.Error { + dumpIndent(w, level) switch n.Type { case ErrorNode: return os.NewError("unexpected ErrorNode") @@ -81,10 +85,15 @@ func dumpLevel(w io.Writer, n *Node, level int) os.Error { return os.NewError("unexpected DocumentNode") case ElementNode: fmt.Fprintf(w, "<%s>", n.Data) + for _, a := range n.Attr { + io.WriteString(w, "\n") + dumpIndent(w, level+1) + fmt.Fprintf(w, `%s="%s"`, a.Key, a.Val) + } case TextNode: fmt.Fprintf(w, "%q", n.Data) case CommentNode: - return os.NewError("COMMENT") + fmt.Fprintf(w, "<!-- %s -->", n.Data) case DoctypeNode: fmt.Fprintf(w, "<!DOCTYPE %s>", n.Data) case scopeMarkerNode: @@ -123,7 +132,7 @@ func TestParser(t *testing.T) { rc := make(chan io.Reader) go readDat(filename, rc) // TODO(nigeltao): Process all test cases, not just a subset. - for i := 0; i < 27; i++ { + for i := 0; i < 34; i++ { // Parse the #data section. b, err := ioutil.ReadAll(<-rc) if err != nil { @@ -152,6 +161,13 @@ func TestParser(t *testing.T) { continue } // Check that rendering and re-parsing results in an identical tree. + if filename == "tests1.dat" && i == 30 { + // Test 30 in tests1.dat is such messed-up markup that a correct parse + // results in a non-conforming tree (one <a> element nested inside another). + // Therefore when it is rendered and re-parsed, it isn't the same. + // So we skip rendering on that test. + continue + } pr, pw := io.Pipe() go func() { pw.CloseWithError(Render(pw, doc)) diff --git a/libgo/go/html/render.go b/libgo/go/html/render.go index e1ec66f..d5dc448 100644 --- a/libgo/go/html/render.go +++ b/libgo/go/html/render.go @@ -30,9 +30,6 @@ type writer interface { // would become a tree containing <html>, <head> and <body> elements. Another // example is that the programmatic equivalent of "a<head>b</head>c" becomes // "<html><head><head/><body>abc</body></html>". -// -// Comment nodes are elided from the output, analogous to Parse skipping over -// any <!--comment--> input. func Render(w io.Writer, n *Node) os.Error { if x, ok := w.(writer); ok { return render(x, n) @@ -61,6 +58,15 @@ func render(w writer, n *Node) os.Error { case ElementNode: // No-op. case CommentNode: + if _, err := w.WriteString("<!--"); err != nil { + return err + } + if _, err := w.WriteString(n.Data); err != nil { + return err + } + if _, err := w.WriteString("-->"); err != nil { + return err + } return nil case DoctypeNode: if _, err := w.WriteString("<!DOCTYPE "); err != nil { diff --git a/libgo/go/html/token.go b/libgo/go/html/token.go index 2826f95..952d174 100644 --- a/libgo/go/html/token.go +++ b/libgo/go/html/token.go @@ -116,10 +116,6 @@ type span struct { // A Tokenizer returns a stream of HTML Tokens. type Tokenizer struct { - // If ReturnComments is set, Next returns comment tokens; - // otherwise it skips over comments (default). - ReturnComments bool - // r is the source of the HTML text. r io.Reader // tt is the TokenType of the current token. @@ -546,17 +542,19 @@ func (z *Tokenizer) readTagAttrVal() { } } -// next scans the next token and returns its type. -func (z *Tokenizer) next() TokenType { +// Next scans the next token and returns its type. +func (z *Tokenizer) Next() TokenType { if z.err != nil { - return ErrorToken + z.tt = ErrorToken + return z.tt } z.raw.start = z.raw.end z.data.start = z.raw.end z.data.end = z.raw.end if z.rawTag != "" { z.readRawOrRCDATA() - return TextToken + z.tt = TextToken + return z.tt } z.textIsRaw = false @@ -596,11 +594,13 @@ loop: if x := z.raw.end - len("<a"); z.raw.start < x { z.raw.end = x z.data.end = x - return TextToken + z.tt = TextToken + return z.tt } switch tokenType { case StartTagToken: - return z.readStartTag() + z.tt = z.readStartTag() + return z.tt case EndTagToken: c = z.readByte() if z.err != nil { @@ -616,39 +616,31 @@ loop: } if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' { z.readEndTag() - return EndTagToken + z.tt = EndTagToken + return z.tt } z.raw.end-- z.readUntilCloseAngle() - return CommentToken + z.tt = CommentToken + return z.tt case CommentToken: if c == '!' { - return z.readMarkupDeclaration() + z.tt = z.readMarkupDeclaration() + return z.tt } z.raw.end-- z.readUntilCloseAngle() - return CommentToken + z.tt = CommentToken + return z.tt } } if z.raw.start < z.raw.end { z.data.end = z.raw.end - return TextToken - } - return ErrorToken -} - -// Next scans the next token and returns its type. -func (z *Tokenizer) Next() TokenType { - for { - z.tt = z.next() - // TODO: remove the ReturnComments option. A tokenizer should - // always return comment tags. - if z.tt == CommentToken && !z.ReturnComments { - continue - } + z.tt = TextToken return z.tt } - panic("unreachable") + z.tt = ErrorToken + return z.tt } // Raw returns the unmodified text of the current token. Calling Next, Token, diff --git a/libgo/go/html/token_test.go b/libgo/go/html/token_test.go index 310cd97..45ce85e 100644 --- a/libgo/go/html/token_test.go +++ b/libgo/go/html/token_test.go @@ -424,7 +424,6 @@ func TestTokenizer(t *testing.T) { loop: for _, tt := range tokenTests { z := NewTokenizer(strings.NewReader(tt.html)) - z.ReturnComments = true if tt.golden != "" { for i, s := range strings.Split(tt.golden, "$") { if z.Next() == ErrorToken { |