}
-func (S *Scanner) scanIdentifier() int {
+func (S *Scanner) scanIdentifier() token.Token {
pos := S.loc.Pos;
for isLetter(S.ch) || isDigit(S.ch) {
S.next();
}
-func (S *Scanner) scanNumber(seen_decimal_point bool) int {
+func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token {
tok := token.INT;
if seen_decimal_point {
// respectively. Otherwise, the result is tok0 if there was no other
// matching character, or tok2 if the matching character was ch2.
-func (S *Scanner) switch2(tok0, tok1 int) int {
+func (S *Scanner) switch2(tok0, tok1 token.Token) token.Token {
if S.ch == '=' {
S.next();
return tok1;
}
-func (S *Scanner) switch3(tok0, tok1, ch2, tok2 int) int {
+func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) token.Token {
if S.ch == '=' {
S.next();
return tok1;
}
-func (S *Scanner) switch4(tok0, tok1, ch2, tok2, tok3 int) int {
+func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Token) token.Token {
if S.ch == '=' {
S.next();
return tok1;
// the token tok, and the literal text lit corresponding to the
// token. The source end is indicated by token.EOF.
//
-func (S *Scanner) Scan() (loc Location, tok int, lit []byte) {
+func (S *Scanner) Scan() (loc Location, tok token.Token, lit []byte) {
scan_again:
// skip white space
for S.ch == ' ' || S.ch == '\t' || S.ch == '\n' || S.ch == '\r' {
// meaning as for the Init function. Tokenize keeps scanning until f returns
// false (usually when the token value is token.EOF).
//
-func Tokenize(src []byte, err ErrorHandler, scan_comments bool, f func (loc Location, tok int, lit []byte) bool) {
+func Tokenize(src []byte, err ErrorHandler, scan_comments bool, f func (loc Location, tok token.Token, lit []byte) bool) {
var s Scanner;
s.Init(src, err, scan_comments);
for f(s.Scan()) {
)
-func tokenclass(tok int) int {
+func tokenclass(tok token.Token) int {
switch {
- case token.IsLiteral(tok): return literal;
- case token.IsOperator(tok): return operator;
- case token.IsKeyword(tok): return keyword;
+ case tok.IsLiteral(): return literal;
+ case tok.IsOperator(): return operator;
+ case tok.IsKeyword(): return keyword;
}
return special;
}
type elt struct {
- tok int;
+ tok token.Token;
lit string;
class int;
}
index := 0;
eloc := scanner.Location{0, 1, 1};
scanner.Tokenize(io.StringBytes(src), &TestErrorHandler{t}, true,
- func (loc Location, tok int, litb []byte) bool {
+ func (loc Location, tok token.Token, litb []byte) bool {
e := elt{token.EOF, "", special};
if index < len(tokens) {
e = tokens[index];
t.Errorf("bad column for %s: got %d, expected %d", lit, loc.Col, eloc.Col);
}
if tok != e.tok {
- t.Errorf("bad token for %s: got %s, expected %s", lit, token.TokenString(tok), token.TokenString(e.tok));
+ t.Errorf("bad token for %s: got %s, expected %s", lit, tok.String(), e.tok.String());
}
- if token.IsLiteral(e.tok) && lit != e.lit {
+ if e.tok.IsLiteral() && lit != e.lit {
t.Errorf("bad literal for %s: got %s, expected %s", lit, lit, e.lit);
}
if tokenclass(tok) != e.class {
import "strconv"
+// Token is the set of lexical tokens of the Go programming language.
+type Token int
+
// The list of tokens.
const (
// Special tokens
- ILLEGAL = iota;
+ ILLEGAL Token = iota;
EOF;
COMMENT;
// At the moment we have no array literal syntax that lets us describe
// the index for each element - use a map for now to make sure they are
// in sync.
-var tokens = map [int] string {
+var tokens = map [Token] string {
ILLEGAL : "ILLEGAL",
EOF : "EOF",
}
-// TokenString returns the string corresponding to the token tok.
+// String returns the string corresponding to the token tok.
// For operators, delimiters, and keywords the string is the actual
// token character sequence (e.g., for the token ADD, the string is
// "+"). For all other tokens the string corresponds to the token
// constant name (e.g. for the token IDENT, the string is "IDENT").
//
-func TokenString(tok int) string {
+func (tok Token) String() string {
if str, exists := tokens[tok]; exists {
return str;
}
// Precedence returns the syntax precedence of the operator
// token tok or LowestPrecedence if tok is not an operator.
//
-func Precedence(tok int) int {
+func (tok Token) Precedence() int {
switch tok {
case COLON:
return 0;
}
-var keywords map [string] int;
+var keywords map [string] Token;
func init() {
- keywords = make(map [string] int);
+ keywords = make(map [string] Token);
for i := keyword_beg + 1; i < keyword_end; i++ {
keywords[tokens[i]] = i;
}
// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
//
-func Lookup(ident []byte) int {
+func Lookup(ident []byte) Token {
// TODO Maps with []byte key are illegal because []byte does not
// support == . Should find a more efficient solution eventually.
if tok, is_keyword := keywords[string(ident)]; is_keyword {
// IsLiteral returns true for tokens corresponding to identifiers
// and basic type literals; returns false otherwise.
//
-func IsLiteral(tok int) bool {
+func (tok Token) IsLiteral() bool {
return literal_beg < tok && tok < literal_end;
}
// IsOperator returns true for tokens corresponding to operators and
// delimiters; returns false otherwise.
//
-func IsOperator(tok int) bool {
+func (tok Token) IsOperator() bool {
return operator_beg < tok && tok < operator_end;
}
// IsKeyword returns true for tokens corresponding to keywords;
// returns false otherwise.
//
-func IsKeyword(tok int) bool {
+func (tok Token) IsKeyword() bool {
return keyword_beg < tok && tok < keyword_end;
}
//
UnaryExpr struct {
Pos_ Position; // token position
- Tok int; // operator
+ Tok token.Token; // operator
X Expr; // operand
};
BinaryExpr struct {
X Expr; // left operand
Pos_ Position; // token position
- Tok int; // operator
+ Tok token.Token; // operator
Y Expr; // right operand
};
)
// An IncDecStmt node represents an increment or decrement statement.
IncDecStmt struct {
X Expr;
- Tok int; // INC or DEC
+ Tok token.Token; // INC or DEC
};
// An AssignStmt node represents an assignment or
AssignStmt struct {
Lhs []Expr;
Pos_ Position; // token position
- Tok int; // assignment token, DEFINE
+ Tok token.Token; // assignment token, DEFINE
Rhs []Expr;
};
//
BranchStmt struct {
Pos_ Position; // position of keyword
- Tok int; // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH)
+ Tok token.Token; // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH)
Label *Ident;
};
// A CommClause node represents a case of a select statement.
CommClause struct {
Case Position; // position of "case" or "default" keyword
- Tok int; // ASSIGN or DEFINE (valid only if Lhs != nil)
+ Tok token.Token; // ASSIGN or DEFINE (valid only if Lhs != nil)
Lhs, Rhs Expr; // Rhs == nil means default case
Colon Position; // position of ":"
Body []Stmt; // statement list; or nil
For Position; // position of "for" keyword
Key, Value Expr; // Value may be nil
Pos_ Position; // token position
- Tok int; // ASSIGN or DEFINE
+ Tok token.Token; // ASSIGN or DEFINE
X Expr; // value to range over
Body *BlockStmt;
};
DeclList struct {
Doc Comments; // associated documentation; or nil
Pos_ Position; // position of token
- Tok int; // IMPORT, CONST, VAR, TYPE
+ Tok token.Token; // IMPORT, CONST, VAR, TYPE
Lparen Position; // position of '('
List []Decl; // the list of parenthesized declarations
Rparen Position; // position of ')'
// The next token
pos Position; // token location
- tok int; // one token look-ahead
+ tok token.Token; // one token look-ahead
val []byte; // token value
// Non-syntactic parser control
P.printIndent();
switch P.tok {
case token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING:
- fmt.Printf("%d:%d: %s = %s\n", P.pos.Line, P.pos.Col, token.TokenString(P.tok), P.val);
+ fmt.Printf("%d:%d: %s = %s\n", P.pos.Line, P.pos.Col, P.tok.String(), P.val);
case token.LPAREN:
// don't print '(' - screws up selection in terminal window
fmt.Printf("%d:%d: LPAREN\n", P.pos.Line, P.pos.Col);
// don't print ')' - screws up selection in terminal window
fmt.Printf("%d:%d: RPAREN\n", P.pos.Line, P.pos.Col);
default:
- fmt.Printf("%d:%d: %s\n", P.pos.Line, P.pos.Col, token.TokenString(P.tok));
+ fmt.Printf("%d:%d: %s\n", P.pos.Line, P.pos.Col, P.tok.String());
}
}
}
}
-func (P *Parser) expect(tok int) Position {
+func (P *Parser) expect(tok token.Token) Position {
if P.tok != tok {
- msg := "expected '" + token.TokenString(tok) + "', found '" + token.TokenString(P.tok) + "'";
- if token.IsLiteral(P.tok) {
+ msg := "expected '" + tok.String() + "', found '" + P.tok.String() + "'";
+ if P.tok.IsLiteral() {
msg += " " + string(P.val);
}
P.error(P.pos, msg);
}
x := P.parseUnaryExpr();
- for prec := token.Precedence(P.tok); prec >= prec1; prec-- {
- for token.Precedence(P.tok) == prec {
+ for prec := P.tok.Precedence(); prec >= prec1; prec-- {
+ for P.tok.Precedence() == prec {
pos, tok := P.pos, P.tok;
P.next();
y := P.parseBinaryExpr(prec + 1);
}
-func (P *Parser) parseBranchStmt(tok int) *ast.BranchStmt {
+func (P *Parser) parseBranchStmt(tok token.Token) *ast.BranchStmt {
if P.trace {
defer un(trace(P, "BranchStmt"));
}
// CommCase
loc := P.pos;
- var tok int;
+ var tok token.Token;
var lhs, rhs ast.Expr;
if P.tok == token.CASE {
P.next();
}
-func (P *Printer) Token(loc scanner.Location, tok int) {
- P.String(loc, token.TokenString(tok));
- //P.TaggedString(pos, "<b>", token.TokenString(tok), "</b>");
+func (P *Printer) Token(loc scanner.Location, tok token.Token) {
+ P.String(loc, tok.String());
+ //P.TaggedString(pos, "<b>", tok.String(), "</b>");
}
-func (P *Printer) Error(loc scanner.Location, tok int, msg string) {
- fmt.Printf("\ninternal printing error: pos = %d, tok = %s, %s\n", loc.Pos, token.TokenString(tok), msg);
+func (P *Printer) Error(loc scanner.Location, tok token.Token, msg string) {
+ fmt.Printf("\ninternal printing error: pos = %d, tok = %s, %s\n", loc.Pos, tok.String(), msg);
panic();
}
func (P *Printer) DoBinaryExpr(x *ast.BinaryExpr) {
- prec := token.Precedence(x.Tok);
+ prec := x.Tok.Precedence();
if prec < P.prec {
P.Token(nopos, token.LPAREN);
}