From: Robert Griesemer Date: Sat, 20 Feb 2016 20:53:34 +0000 (-0800) Subject: cmd/compile: set lexer nlsemi state directly X-Git-Tag: go1.7beta1~1799 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=aa5b44aeabccd25a320d6ea609c41781bdbf5ce0;p=gostls13.git cmd/compile: set lexer nlsemi state directly The old code used an extra function call and switch to inspect the current token and determine the new state of curio.nlsemi. However, the lexer knows the token w/o the need of an extra test and thus can set curio.nlsemi directly: - removed need for extra function call in next - renamed _yylex to next - set nlsemi at the point a token is identified - moved nlsemi from curio to lexer - it's really part of the lexer state This change makes the lexer call sequence less convoluted and should also speed up the lexing a bit. Change-Id: Iaf2683081f04231cb62c94e1400d455f98f6f82a Reviewed-on: https://go-review.googlesource.com/19765 Reviewed-by: Matthew Dempsky --- diff --git a/src/cmd/compile/internal/gc/go.go b/src/cmd/compile/internal/gc/go.go index 9b55eb8848..d21b2fb196 100644 --- a/src/cmd/compile/internal/gc/go.go +++ b/src/cmd/compile/internal/gc/go.go @@ -389,7 +389,6 @@ type Io struct { last int peekc int peekc1 int // second peekc for ... - nlsemi bool eofnl bool } diff --git a/src/cmd/compile/internal/gc/lex.go b/src/cmd/compile/internal/gc/lex.go index 55d988b70a..54c69c5449 100644 --- a/src/cmd/compile/internal/gc/lex.go +++ b/src/cmd/compile/internal/gc/lex.go @@ -869,6 +869,7 @@ func isfrog(c int) bool { type lexer struct { // TODO(gri) move other lexer state here and out of global variables // (source, current line number, etc.) + nlsemi bool // if set, '\n' and EOF translate to ';' // current token tok int32 @@ -925,7 +926,7 @@ const ( LRSH ) -func (yylval *lexer) _yylex() int32 { +func (l *lexer) next() { var c1 int var op Op var escflag int @@ -936,27 +937,31 @@ func (yylval *lexer) _yylex() int32 { prevlineno = lineno + nlsemi := l.nlsemi + l.nlsemi = false + l0: + // skip white space c := getc() - if isSpace(c) { - if c == '\n' && curio.nlsemi { + for isSpace(c) { + if c == '\n' && nlsemi { ungetc(c) if Debug['x'] != 0 { fmt.Printf("lex: implicit semi\n") } - return ';' + l.tok = ';' + return } - - goto l0 + c = getc() } - lineno = lexlineno // start of token + // start of token + lineno = lexlineno if c >= utf8.RuneSelf { // all multibyte runes are alpha cp = &lexbuf cp.Reset() - goto talph } @@ -1049,7 +1054,17 @@ l0: case EOF: lineno = prevlineno ungetc(EOF) - return -1 + // Treat EOF as "end of line" for the purposes + // of inserting a semicolon. + if nlsemi { + if Debug['x'] != 0 { + fmt.Printf("lex: implicit semi\n") + } + l.tok = ';' + return + } + l.tok = -1 + return case '_': cp = &lexbuf @@ -1137,14 +1152,16 @@ l0: } x := new(Mpint) - yylval.val.U = x + l.val.U = x Mpmovecfix(x, v) x.Rune = true if Debug['x'] != 0 { fmt.Printf("lex: codepoint literal\n") } litbuf = "string literal" - return LLITERAL + l.nlsemi = true + l.tok = LLITERAL + return case '/': c1 = getc() @@ -1217,6 +1234,7 @@ l0: case '+': c1 = getc() if c1 == '+' { + l.nlsemi = true c = int(LINC) goto lx } @@ -1229,6 +1247,7 @@ l0: case '-': c1 = getc() if c1 == '-' { + l.nlsemi = true c = int(LDEC) goto lx } @@ -1339,6 +1358,10 @@ l0: goto asop } + case ')', ']', '}': + l.nlsemi = true + goto lx + default: goto lx } @@ -1363,14 +1386,16 @@ lx: goto l0 } - return int32(c) + l.tok = int32(c) + return asop: - yylval.op = op + l.op = op if Debug['x'] != 0 { fmt.Printf("lex: TOKEN ASOP %s=\n", goopnames[op]) } - return LASOP + l.tok = LASOP + return // cp is set to lexbuf and some // prefix has been stored @@ -1407,26 +1432,33 @@ talph: if Debug['x'] != 0 { fmt.Printf("lex: %s %s\n", s, lexname(int(s.Lexical))) } - yylval.sym_ = s - return int32(s.Lexical) + l.sym_ = s + switch s.Lexical { + case LNAME, LRETURN, LBREAK, LCONTINUE, LFALL: + l.nlsemi = true + } + l.tok = int32(s.Lexical) + return ncu: cp = nil ungetc(c) str = lexbuf.String() - yylval.val.U = new(Mpint) - mpatofix(yylval.val.U.(*Mpint), str) - if yylval.val.U.(*Mpint).Ovf { + l.val.U = new(Mpint) + mpatofix(l.val.U.(*Mpint), str) + if l.val.U.(*Mpint).Ovf { Yyerror("overflow in constant") - Mpmovecfix(yylval.val.U.(*Mpint), 0) + Mpmovecfix(l.val.U.(*Mpint), 0) } if Debug['x'] != 0 { fmt.Printf("lex: integer literal\n") } litbuf = "literal " + str - return LLITERAL + l.nlsemi = true + l.tok = LLITERAL + return casedot: for { @@ -1475,45 +1507,50 @@ casei: cp = nil str = lexbuf.String() - yylval.val.U = new(Mpcplx) - Mpmovecflt(&yylval.val.U.(*Mpcplx).Real, 0.0) - mpatoflt(&yylval.val.U.(*Mpcplx).Imag, str) - if yylval.val.U.(*Mpcplx).Imag.Val.IsInf() { + l.val.U = new(Mpcplx) + Mpmovecflt(&l.val.U.(*Mpcplx).Real, 0.0) + mpatoflt(&l.val.U.(*Mpcplx).Imag, str) + if l.val.U.(*Mpcplx).Imag.Val.IsInf() { Yyerror("overflow in imaginary constant") - Mpmovecflt(&yylval.val.U.(*Mpcplx).Imag, 0.0) + Mpmovecflt(&l.val.U.(*Mpcplx).Imag, 0.0) } if Debug['x'] != 0 { fmt.Printf("lex: imaginary literal\n") } litbuf = "literal " + str - return LLITERAL + l.nlsemi = true + l.tok = LLITERAL + return caseout: cp = nil ungetc(c) str = lexbuf.String() - yylval.val.U = newMpflt() - mpatoflt(yylval.val.U.(*Mpflt), str) - if yylval.val.U.(*Mpflt).Val.IsInf() { + l.val.U = newMpflt() + mpatoflt(l.val.U.(*Mpflt), str) + if l.val.U.(*Mpflt).Val.IsInf() { Yyerror("overflow in float constant") - Mpmovecflt(yylval.val.U.(*Mpflt), 0.0) + Mpmovecflt(l.val.U.(*Mpflt), 0.0) } if Debug['x'] != 0 { fmt.Printf("lex: floating literal\n") } litbuf = "literal " + str - return LLITERAL + l.nlsemi = true + l.tok = LLITERAL + return strlit: - yylval.val.U = internString(cp.Bytes()) + l.val.U = internString(cp.Bytes()) if Debug['x'] != 0 { fmt.Printf("lex: string literal\n") } litbuf = "string literal" - return LLITERAL + l.nlsemi = true + l.tok = LLITERAL } var internedStrings = map[string]string{} @@ -1833,36 +1870,6 @@ func pragcgo(text string) { } } -func (l *lexer) next() { - tok := l._yylex() - - if curio.nlsemi && tok == EOF { - // Treat EOF as "end of line" for the purposes - // of inserting a semicolon. - tok = ';' - } - - switch tok { - case LNAME, - LLITERAL, - LBREAK, - LCONTINUE, - LFALL, - LRETURN, - LINC, - LDEC, - ')', - '}', - ']': - curio.nlsemi = true - - default: - curio.nlsemi = false - } - - l.tok = tok -} - func getc() int { c := curio.peekc if c != 0 {