itemRawString // raw quoted string (includes quotes)
itemRightDelim // right action delimiter
itemRightParen // ')' inside action
+ itemSpace // run of spaces separating arguments
itemString // quoted string (includes quotes)
itemText // plain text
itemVariable // variable starting with '$', such as '$' or '$1' or '$hello'.
itemWith // with keyword
)
-// Make the types prettyprint.
-var itemName = map[itemType]string{
- itemError: "error",
- itemBool: "bool",
- itemChar: "char",
- itemCharConstant: "charconst",
- itemComplex: "complex",
- itemColonEquals: ":=",
- itemEOF: "EOF",
- itemField: "field",
- itemIdentifier: "identifier",
- itemLeftDelim: "left delim",
- itemLeftParen: "(",
- itemNumber: "number",
- itemPipe: "pipe",
- itemRawString: "raw string",
- itemRightDelim: "right delim",
- itemRightParen: ")",
- itemString: "string",
- itemVariable: "variable",
-
- // keywords
- itemDot: ".",
- itemDefine: "define",
- itemElse: "else",
- itemIf: "if",
- itemEnd: "end",
- itemNil: "nil",
- itemRange: "range",
- itemTemplate: "template",
- itemWith: "with",
-}
-
-func (i itemType) String() string {
- s := itemName[i]
- if s == "" {
- return fmt.Sprintf("item%d", int(i))
- }
- return s
-}
-
var key = map[string]itemType{
".": itemDot,
"define": itemDefine,
// lexInsideAction scans the elements inside action delimiters.
func lexInsideAction(l *lexer) stateFn {
// Either number, quoted string, or identifier.
- // Spaces separate and are ignored.
+ // Spaces separate arguments; runs of spaces turn into itemSpace.
// Pipe symbols separate and are emitted.
if strings.HasPrefix(l.input[l.pos:], l.rightDelim) {
if l.parenDepth == 0 {
return l.errorf("unclosed left paren")
}
switch r := l.next(); {
- case r == eof || r == '\n':
+ case r == eof || isEndOfLine(r):
return l.errorf("unclosed action")
case isSpace(r):
- l.ignore()
+ return lexSpace
case r == ':':
if l.next() != '=' {
return l.errorf("expected :=")
if l.parenDepth < 0 {
return l.errorf("unexpected right paren %#U", r)
}
- // Catch the mistake of (a).X, which will parse as two args.
- // See issue 3999. TODO: Remove once arg parsing is
- // better defined.
- if l.peek() == '.' {
- return l.errorf("cannot evaluate field of parenthesized expression")
- }
return lexInsideAction
case r <= unicode.MaxASCII && unicode.IsPrint(r):
l.emit(itemChar)
return lexInsideAction
}
+// lexSpace scans a run of space characters.
+// One space has already been seen.
+func lexSpace(l *lexer) stateFn {
+ for isSpace(l.peek()) {
+ l.next()
+ }
+ l.emit(itemSpace)
+ return lexInsideAction
+}
+
// lexIdentifier scans an alphanumeric or field.
func lexIdentifier(l *lexer) stateFn {
Loop:
// arithmetic.
func (l *lexer) atTerminator() bool {
r := l.peek()
- if isSpace(r) {
+ if isSpace(r) || isEndOfLine(r) {
return true
}
switch r {
// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
- switch r {
- case ' ', '\t', '\n', '\r':
- return true
- }
- return false
+ return r == ' ' || r == '\t'
+}
+
+// isEndOfLine reports whether r is an end-of-line character
+func isEndOfLine(r rune) bool {
+ return r == '\r' || r == '\n'
}
// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
package parse
import (
+ "fmt"
"testing"
)
+// Make the types prettyprint.
+var itemName = map[itemType]string{
+ itemError: "error",
+ itemBool: "bool",
+ itemChar: "char",
+ itemCharConstant: "charconst",
+ itemComplex: "complex",
+ itemColonEquals: ":=",
+ itemEOF: "EOF",
+ itemField: "field",
+ itemIdentifier: "identifier",
+ itemLeftDelim: "left delim",
+ itemLeftParen: "(",
+ itemNumber: "number",
+ itemPipe: "pipe",
+ itemRawString: "raw string",
+ itemRightDelim: "right delim",
+ itemRightParen: ")",
+ itemSpace: "space",
+ itemString: "string",
+ itemVariable: "variable",
+
+ // keywords
+ itemDot: ".",
+ itemDefine: "define",
+ itemElse: "else",
+ itemIf: "if",
+ itemEnd: "end",
+ itemNil: "nil",
+ itemRange: "range",
+ itemTemplate: "template",
+ itemWith: "with",
+}
+
+func (i itemType) String() string {
+ s := itemName[i]
+ if s == "" {
+ return fmt.Sprintf("item%d", int(i))
+ }
+ return s
+}
+
type lexTest struct {
name string
input string
var (
tEOF = item{itemEOF, 0, ""}
+ tFor = item{itemIdentifier, 0, "for"}
tLeft = item{itemLeftDelim, 0, "{{"}
- tRight = item{itemRightDelim, 0, "}}"}
- tRange = item{itemRange, 0, "range"}
tPipe = item{itemPipe, 0, "|"}
- tFor = item{itemIdentifier, 0, "for"}
tQuote = item{itemString, 0, `"abc \n\t\" "`}
+ tRange = item{itemRange, 0, "range"}
+ tRight = item{itemRightDelim, 0, "}}"}
+ tSpace = item{itemSpace, 0, " "}
raw = "`" + `abc\n\t\" ` + "`"
tRawQuote = item{itemRawString, 0, raw}
)
{itemText, 0, "-world"},
tEOF,
}},
- {"punctuation", "{{,@%}}", []item{
+ {"punctuation", "{{,@% }}", []item{
tLeft,
{itemChar, 0, ","},
{itemChar, 0, "@"},
{itemChar, 0, "%"},
+ tSpace,
tRight,
tEOF,
}},
tEOF,
}},
{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
- {"for", `{{for }}`, []item{tLeft, tFor, tRight, tEOF}},
+ {"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
{"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{
tLeft,
{itemNumber, 0, "1"},
+ tSpace,
{itemNumber, 0, "02"},
+ tSpace,
{itemNumber, 0, "0x14"},
+ tSpace,
{itemNumber, 0, "-7.2i"},
+ tSpace,
{itemNumber, 0, "1e3"},
+ tSpace,
{itemNumber, 0, "+1.2e-4"},
+ tSpace,
{itemNumber, 0, "4.2i"},
+ tSpace,
{itemComplex, 0, "1+2i"},
tRight,
tEOF,
{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
tLeft,
{itemCharConstant, 0, `'a'`},
+ tSpace,
{itemCharConstant, 0, `'\n'`},
+ tSpace,
{itemCharConstant, 0, `'\''`},
+ tSpace,
{itemCharConstant, 0, `'\\'`},
+ tSpace,
{itemCharConstant, 0, `'\u00FF'`},
+ tSpace,
{itemCharConstant, 0, `'\xFF'`},
+ tSpace,
{itemCharConstant, 0, `'本'`},
tRight,
tEOF,
{"bools", "{{true false}}", []item{
tLeft,
{itemBool, 0, "true"},
+ tSpace,
{itemBool, 0, "false"},
tRight,
tEOF,
{"dots", "{{.x . .2 .x.y}}", []item{
tLeft,
{itemField, 0, ".x"},
+ tSpace,
{itemDot, 0, "."},
+ tSpace,
{itemNumber, 0, ".2"},
+ tSpace,
{itemField, 0, ".x.y"},
tRight,
tEOF,
{"keywords", "{{range if else end with}}", []item{
tLeft,
{itemRange, 0, "range"},
+ tSpace,
{itemIf, 0, "if"},
+ tSpace,
{itemElse, 0, "else"},
+ tSpace,
{itemEnd, 0, "end"},
+ tSpace,
{itemWith, 0, "with"},
tRight,
tEOF,
{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
tLeft,
{itemVariable, 0, "$c"},
+ tSpace,
{itemColonEquals, 0, ":="},
+ tSpace,
{itemIdentifier, 0, "printf"},
+ tSpace,
{itemVariable, 0, "$"},
+ tSpace,
{itemVariable, 0, "$hello"},
+ tSpace,
{itemVariable, 0, "$23"},
+ tSpace,
{itemVariable, 0, "$"},
+ tSpace,
{itemVariable, 0, "$var.Field"},
+ tSpace,
{itemField, 0, ".Method"},
tRight,
tEOF,
}},
+ {"variable invocation ", "{{$x 23}}", []item{
+ tLeft,
+ {itemVariable, 0, "$x"},
+ tSpace,
+ {itemNumber, 0, "23"},
+ tRight,
+ tEOF,
+ }},
{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
{itemText, 0, "intro "},
tLeft,
{itemIdentifier, 0, "echo"},
+ tSpace,
{itemIdentifier, 0, "hi"},
+ tSpace,
{itemNumber, 0, "1.2"},
+ tSpace,
tPipe,
{itemIdentifier, 0, "noargs"},
tPipe,
{itemIdentifier, 0, "args"},
+ tSpace,
{itemNumber, 0, "1"},
+ tSpace,
{itemString, 0, `"hi"`},
tRight,
{itemText, 0, " outro"},
{"declaration", "{{$v := 3}}", []item{
tLeft,
{itemVariable, 0, "$v"},
+ tSpace,
{itemColonEquals, 0, ":="},
+ tSpace,
{itemNumber, 0, "3"},
tRight,
tEOF,
{"2 declarations", "{{$v , $w := 3}}", []item{
tLeft,
{itemVariable, 0, "$v"},
+ tSpace,
{itemChar, 0, ","},
+ tSpace,
{itemVariable, 0, "$w"},
+ tSpace,
{itemColonEquals, 0, ":="},
+ tSpace,
{itemNumber, 0, "3"},
tRight,
tEOF,
for _, test := range lexTests {
items := collect(&test, "", "")
if !equal(items, test.items, false) {
- t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
+ t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
}
}
}
tEOF,
}},
{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
- {"for", `$$for @@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
+ {"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
}
// Parsing only; cleared after parse.
funcs []map[string]interface{}
lex *lexer
- token [2]item // two-token lookahead for parser.
+ token [3]item // three-token lookahead for parser.
peekCount int
vars []string // variables defined at the moment.
}
t.peekCount++
}
-// backup2 backs the input stream up two tokens
+// backup2 backs the input stream up two tokens.
+// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
t.token[1] = t1
t.peekCount = 2
}
+// backup3 backs the input stream up three tokens
+// The zeroth token is already there.
+func (t *Tree) backup3(t2, t1 item) { // Reverse order: we're pushing back.
+ t.token[1] = t1
+ t.token[2] = t2
+ t.peekCount = 3
+}
+
// peek returns but does not consume the next token.
func (t *Tree) peek() item {
if t.peekCount > 0 {
return t.token[0]
}
+// nextNonSpace returns the next non-space token.
+func (t *Tree) nextNonSpace() (token item) {
+ for {
+ token = t.next()
+ if token.typ != itemSpace {
+ break
+ }
+ }
+ return token
+}
+
+// peekNonSpace returns but does not consume the next non-space token.
+func (t *Tree) peekNonSpace() (token item) {
+ for {
+ token = t.next()
+ if token.typ != itemSpace {
+ break
+ }
+ }
+ t.backup()
+ return token
+}
+
// Parsing.
// New allocates a new parse tree with the given name.
// expect consumes the next token and guarantees it has the required type.
func (t *Tree) expect(expected itemType, context string) item {
- token := t.next()
+ token := t.nextNonSpace()
if token.typ != expected {
t.errorf("expected %s in %s; got %s", expected, context, token)
}
// expectOneOf consumes the next token and guarantees it has one of the required types.
func (t *Tree) expectOneOf(expected1, expected2 itemType, context string) item {
- token := t.next()
+ token := t.nextNonSpace()
if token.typ != expected1 && token.typ != expected2 {
t.errorf("expected %s or %s in %s; got %s", expected1, expected2, context, token)
}
for t.peek().typ != itemEOF {
if t.peek().typ == itemLeftDelim {
delim := t.next()
- if t.next().typ == itemDefine {
+ if t.nextNonSpace().typ == itemDefine {
newT := New("definition") // name will be updated once we know it.
newT.startParse(t.funcs, t.lex)
newT.parseDefinition(treeSet)
// Terminates at {{end}} or {{else}}, returned separately.
func (t *Tree) itemList() (list *ListNode, next Node) {
list = newList()
- for t.peek().typ != itemEOF {
+ for t.peekNonSpace().typ != itemEOF {
n := t.textOrAction()
switch n.Type() {
case nodeEnd, nodeElse:
// textOrAction:
// text | action
func (t *Tree) textOrAction() Node {
- switch token := t.next(); token.typ {
+ switch token := t.nextNonSpace(); token.typ {
case itemText:
return newText(token.val)
case itemLeftDelim:
// Left delim is past. Now get actions.
// First word could be a keyword such as range.
func (t *Tree) action() (n Node) {
- switch token := t.next(); token.typ {
+ switch token := t.nextNonSpace(); token.typ {
case itemElse:
return t.elseControl()
case itemEnd:
var decl []*VariableNode
// Are there declarations?
for {
- if v := t.peek(); v.typ == itemVariable {
+ if v := t.peekNonSpace(); v.typ == itemVariable {
t.next()
- if next := t.peek(); next.typ == itemColonEquals || (next.typ == itemChar && next.val == ",") {
- t.next()
+ // Since space is a token, we need 3-token look-ahead here in the worst case:
+ // in "$x foo" we need to read "foo" (as opposed to ":=") to know that $x is an
+ // argument variable rather than a declaration. So remember the token
+ // adjacent to the variable so we can push it back if necessary.
+ tokenAfterVariable := t.peek()
+ if next := t.peekNonSpace(); next.typ == itemColonEquals || (next.typ == itemChar && next.val == ",") {
+ t.nextNonSpace()
variable := newVariable(v.val)
if len(variable.Ident) != 1 {
t.errorf("illegal variable in declaration: %s", v.val)
}
t.errorf("too many declarations in %s", context)
}
+ } else if tokenAfterVariable.typ == itemSpace {
+ t.backup3(v, tokenAfterVariable)
} else {
t.backup2(v)
}
}
pipe = newPipeline(t.lex.lineNumber(), decl)
for {
- switch token := t.next(); token.typ {
+ switch token := t.nextNonSpace(); token.typ {
case itemRightDelim, itemRightParen:
if len(pipe.Cmds) == 0 {
t.errorf("missing value for %s", context)
// to a string.
func (t *Tree) templateControl() Node {
var name string
- switch token := t.next(); token.typ {
+ switch token := t.nextNonSpace(); token.typ {
case itemString, itemRawString:
s, err := strconv.Unquote(token.val)
if err != nil {
t.unexpected(token, "template invocation")
}
var pipe *PipeNode
- if t.next().typ != itemRightDelim {
+ if t.nextNonSpace().typ != itemRightDelim {
t.backup()
// Do not pop variables; they persist until "end".
pipe = t.pipeline("template")
cmd := newCommand()
Loop:
for {
- switch token := t.next(); token.typ {
+ switch token := t.nextNonSpace(); token.typ {
case itemRightDelim, itemRightParen:
t.backup()
break Loop
break Loop
case itemLeftParen:
p := t.pipeline("parenthesized expression")
- if t.next().typ != itemRightParen {
+ if t.nextNonSpace().typ != itemRightParen {
t.errorf("missing right paren in parenthesized expression")
}
cmd.append(p)
default:
t.unexpected(token, "command")
}
+ t.terminate()
}
if len(cmd.Args) == 0 {
t.errorf("empty command")
return cmd
}
+// terminate checks that the next token terminates an argument. This guarantees
+// that arguments are space-separated, for example that (2)3 does not parse.
+func (t *Tree) terminate() {
+ token := t.peek()
+ switch token.typ {
+ case itemChar, itemPipe, itemRightDelim, itemRightParen, itemSpace:
+ return
+ }
+ t.unexpected(token, "argument list (missing space?)")
+}
+
// hasFunction reports if a function name exists in the Tree's maps.
func (t *Tree) hasFunction(name string) bool {
for _, funcMap := range t.funcs {
{"invalid punctuation", "{{printf 3, 4}}", hasError, ""},
{"multidecl outside range", "{{with $v, $u := 3}}{{end}}", hasError, ""},
{"too many decls in range", "{{range $u, $v, $w := 3}}{{end}}", hasError, ""},
- // This one should work but doesn't. Caught as a parse error to avoid confusion.
- // TODO: Update after issue 3999 is resolved.
{"dot applied to parentheses", "{{printf (printf .).}}", hasError, ""},
+ {"adjacent args", "{{printf 3`x`}}", hasError, ""},
+ {"adjacent args with .", "{{printf `x`.}}", hasError, ""},
// Equals (and other chars) do not assignments make (yet).
{"bug0a", "{{$x := 0}}{{$x}}", noError, "{{$x := 0}}{{$x}}"},
{"bug0b", "{{$x = 1}}{{$x}}", hasError, ""},