SelfClosingTagToken
// A CommentToken looks like <!--x-->.
CommentToken
+ // A DoctypeToken looks like <!DOCTYPE x>
+ DoctypeToken
)
// String returns a string representation of the TokenType.
return "SelfClosingTag"
case CommentToken:
return "Comment"
+ case DoctypeToken:
+ return "Doctype"
}
return "Invalid(" + strconv.Itoa(int(t)) + ")"
}
}
// A Token consists of a TokenType and some Data (tag name for start and end
-// tags, content for text and comments). A tag Token may also contain a slice
-// of Attributes. Data is unescaped for all Tokens (it looks like "a<b" rather
-// than "a<b").
+// tags, content for text, comments and doctypes). A tag Token may also contain
+// a slice of Attributes. Data is unescaped for all Tokens (it looks like "a<b"
+// rather than "a<b").
type Token struct {
Type TokenType
Data string
return "<" + t.tagString() + "/>"
case CommentToken:
return "<!--" + EscapeString(t.Data) + "-->"
+ case DoctypeToken:
+ return "<!DOCTYPE " + EscapeString(t.Data) + ">"
}
return "Invalid(" + strconv.Itoa(int(t.Type)) + ")"
}
// r is the source of the HTML text.
r io.Reader
- // tt is the TokenType of the most recently read token. If tt == Error
- // then err is the error associated with trying to read that token.
- tt TokenType
+ // tt is the TokenType of the most recently read token.
+ tt TokenType
+ // err is the first error encountered during tokenization. It is possible
+ // for tt != Error && err != nil to hold: this means that Next returned a
+ // valid token but the subsequent Next call will return an error token.
+ // For example, if the HTML text input was just "plain", then the first
+ // Next call would set z.err to os.EOF but return a TextToken, and all
+ // subsequent Next calls would return an ErrorToken.
+ // err is never reset. Once it becomes non-nil, it stays non-nil.
err os.Error
// buf[p0:p1] holds the raw data of the most recent token.
// buf[p1:] is buffered input that will yield future tokens.
// readByte returns the next byte from the input stream, doing a buffered read
// from z.r into z.buf if necessary. z.buf[z.p0:z.p1] remains a contiguous byte
// slice that holds all the bytes read so far for the current token.
-func (z *Tokenizer) readByte() (byte, os.Error) {
+// It sets z.err if the underlying reader returns an error.
+// Pre-condition: z.err == nil.
+func (z *Tokenizer) readByte() byte {
if z.p1 >= len(z.buf) {
// Our buffer is exhausted and we have to read from z.r.
// We copy z.buf[z.p0:z.p1] to the beginning of z.buf. If the length
if 2*d > c {
buf1 = make([]byte, d, 2*c)
} else {
- buf1 = z.buf[0:d]
+ buf1 = z.buf[:d]
}
copy(buf1, z.buf[z.p0:z.p1])
- z.p0, z.p1, z.buf = 0, d, buf1[0:d]
+ z.p0, z.p1, z.buf = 0, d, buf1[:d]
// Now that we have copied the live bytes to the start of the buffer,
// we read from z.r into the remainder.
n, err := z.r.Read(buf1[d:cap(buf1)])
if err != nil {
- return 0, err
+ z.err = err
+ return 0
}
- z.buf = buf1[0 : d+n]
+ z.buf = buf1[:d+n]
}
x := z.buf[z.p1]
z.p1++
- return x, nil
+ return x
}
-// readTo keeps reading bytes until x is found.
-func (z *Tokenizer) readTo(x uint8) os.Error {
+// readTo keeps reading bytes until x is found or a read error occurs. If an
+// error does occur, z.err is set to that error.
+// Pre-condition: z.err == nil.
+func (z *Tokenizer) readTo(x uint8) {
for {
- c, err := z.readByte()
- if err != nil {
- return err
+ c := z.readByte()
+ if z.err != nil {
+ return
}
switch c {
case x:
- return nil
+ return
case '\\':
- _, err = z.readByte()
- if err != nil {
- return err
+ z.readByte()
+ if z.err != nil {
+ return
}
}
}
- panic("unreachable")
}
-// nextMarkupDeclaration returns the next TokenType starting with "<!".
-func (z *Tokenizer) nextMarkupDeclaration() (TokenType, os.Error) {
- // TODO: check for <!DOCTYPE ... >, don't just assume that it's a comment.
- for i := 0; i < 2; i++ {
- c, err := z.readByte()
- if err != nil {
- return TextToken, err
- }
- if c != '-' {
- return z.nextText(), nil
- }
- }
+// nextComment reads the next token starting with "<!--".
+// The opening "<!--" has already been consumed.
+// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 4 <= z.p1.
+func (z *Tokenizer) nextComment() {
// <!--> is a valid comment.
for dashCount := 2; ; {
- c, err := z.readByte()
- if err != nil {
- return TextToken, err
+ c := z.readByte()
+ if z.err != nil {
+ return
}
switch c {
case '-':
dashCount++
case '>':
if dashCount >= 2 {
- return CommentToken, nil
+ z.tt = CommentToken
+ return
}
- fallthrough
+ dashCount = 0
default:
dashCount = 0
}
}
- panic("unreachable")
}
-// nextTag returns the next TokenType starting from the tag open state.
-func (z *Tokenizer) nextTag() (tt TokenType, err os.Error) {
- c, err := z.readByte()
- if err != nil {
- return ErrorToken, err
+// nextMarkupDeclaration reads the next token starting with "<!".
+// It might be a "<!--comment-->", a "<!DOCTYPE foo>", or "<!malformed text".
+// The opening "<!" has already been consumed.
+// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 2 <= z.p1.
+func (z *Tokenizer) nextMarkupDeclaration() {
+ var c [2]byte
+ for i := 0; i < 2; i++ {
+ c[i] = z.readByte()
+ if z.err != nil {
+ return
+ }
+ }
+ if c[0] == '-' && c[1] == '-' {
+ z.nextComment()
+ return
+ }
+ z.p1 -= 2
+ const s = "DOCTYPE "
+ for i := 0; ; i++ {
+ c := z.readByte()
+ if z.err != nil {
+ return
+ }
+ // Capitalize c.
+ if 'a' <= c && c <= 'z' {
+ c = 'A' + (c - 'a')
+ }
+ if i < len(s) && c != s[i] {
+ z.nextText()
+ return
+ }
+ if c == '>' {
+ if i >= len(s) {
+ z.tt = DoctypeToken
+ }
+ return
+ }
+ }
+}
+
+// nextTag reads the next token starting with "<". It might be a "<startTag>",
+// an "</endTag>", a "<!markup declaration>", or "<malformed text".
+// The opening "<" has already been consumed.
+// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 1 <= z.p1.
+func (z *Tokenizer) nextTag() {
+ c := z.readByte()
+ if z.err != nil {
+ return
}
+ var tt TokenType
switch {
case c == '/':
tt = EndTagToken
case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z':
tt = StartTagToken
case c == '!':
- return z.nextMarkupDeclaration()
+ z.nextMarkupDeclaration()
+ return
case c == '?':
- return ErrorToken, os.NewError("html: TODO(nigeltao): implement XML processing instructions")
+ z.tt, z.err = ErrorToken, os.NewError("html: TODO: implement XML processing instructions")
+ return
default:
- return ErrorToken, os.NewError("html: TODO(nigeltao): handle malformed tags")
+ z.tt, z.err = ErrorToken, os.NewError("html: TODO: handle malformed tags")
+ return
}
for {
- c, err := z.readByte()
- if err != nil {
- return TextToken, err
+ c := z.readByte()
+ if z.err != nil {
+ return
}
switch c {
- case '"':
- err = z.readTo('"')
- if err != nil {
- return TextToken, err
- }
- case '\'':
- err = z.readTo('\'')
- if err != nil {
- return TextToken, err
+ case '"', '\'':
+ z.readTo(c)
+ if z.err != nil {
+ return
}
case '>':
+ z.tt = tt
if z.buf[z.p1-2] == '/' && tt == StartTagToken {
- return SelfClosingTagToken, nil
+ z.tt = SelfClosingTagToken
}
- return tt, nil
+ return
}
}
- panic("unreachable")
}
// nextText reads all text up until an '<'.
-func (z *Tokenizer) nextText() TokenType {
+// Pre-condition: z.tt == TextToken && z.err == nil && z.p0 + 1 <= z.p1.
+func (z *Tokenizer) nextText() {
for {
- c, err := z.readByte()
- if err != nil {
- z.tt, z.err = ErrorToken, err
- if err == os.EOF {
- z.tt = TextToken
- }
- return z.tt
+ c := z.readByte()
+ if z.err != nil {
+ return
}
if c == '<' {
z.p1--
- z.tt = TextToken
- return z.tt
+ return
}
}
- panic("unreachable")
}
// Next scans the next token and returns its type.
return z.tt
}
z.p0 = z.p1
- c, err := z.readByte()
- if err != nil {
- z.tt, z.err = ErrorToken, err
+ c := z.readByte()
+ if z.err != nil {
+ z.tt = ErrorToken
return z.tt
}
- if c == '<' {
- z.tt, z.err = z.nextTag()
+ // We assume that the next token is text unless proven otherwise.
+ z.tt = TextToken
+ if c != '<' {
+ z.nextText()
+ } else {
+ z.nextTag()
if z.tt == CommentToken && !z.ReturnComments {
continue
}
- return z.tt
}
- return z.nextText()
+ return z.tt
}
panic("unreachable")
}
return z.buf[i0:i], z.trim(i)
}
-// Text returns the unescaped text of a TextToken or a CommentToken.
-// The contents of the returned slice may change on the next call to Next.
+// Text returns the unescaped text of a text, comment or doctype token. The
+// contents of the returned slice may change on the next call to Next.
func (z *Tokenizer) Text() []byte {
+ var i0, i1 int
switch z.tt {
case TextToken:
- s := unescape(z.Raw())
- z.p0 = z.p1
- return s
+ i0 = z.p0
+ i1 = z.p1
case CommentToken:
- // We trim the "<!--" from the left and the "-->" from the right.
+ // Trim the "<!--" from the left and the "-->" from the right.
// "<!-->" is a valid comment, so the adjusted endpoints might overlap.
- i0 := z.p0 + 4
- i1 := z.p1 - 3
- z.p0 = z.p1
- var s []byte
- if i0 < i1 {
- s = unescape(z.buf[i0:i1])
- }
- return s
+ i0 = z.p0 + 4
+ i1 = z.p1 - 3
+ case DoctypeToken:
+ // Trim the "<!DOCTYPE " from the left and the ">" from the right.
+ i0 = z.p0 + 10
+ i1 = z.p1 - 1
+ default:
+ return nil
+ }
+ z.p0 = z.p1
+ if i0 < i1 {
+ return unescape(z.buf[i0:i1])
}
return nil
}
func (z *Tokenizer) Token() Token {
t := Token{Type: z.tt}
switch z.tt {
- case TextToken, CommentToken:
+ case TextToken, CommentToken, DoctypeToken:
t.Data = string(z.Text())
case StartTagToken, EndTagToken, SelfClosingTagToken:
var attr []Attribute