From: Daniel Martí Date: Tue, 20 Feb 2018 10:21:41 +0000 (+0000) Subject: cmd/compile/internal/syntax: use stringer for operators and tokens X-Git-Tag: go1.11beta1~1466 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=c8791538315e633a41461173964a99bd90e103e3;p=gostls13.git cmd/compile/internal/syntax: use stringer for operators and tokens With its new -linecomment flag, it is now possible to use stringer on values whose strings aren't valid identifiers. This is the case with tokens and operators in Go. Operator alredy had inline comments with each operator's string representation; only minor modifications were needed. The inline comments were added to each of the token names, using the same strategy. Comments that were previously inline or part of the string arrays were moved to the line immediately before the name they correspond to. Finally, declare tokStrFast as a function that uses the generated arrays directly. Avoiding the branch and strconv call means that we avoid a performance regression in the scanner, perhaps due to the lack of mid-stack inlining. Performance is not affected. Measured with 'go test -run StdLib -fast' on an X1 Carbon Gen2 (i5-4300U @ 1.90GHz, 8GB RAM, SSD), the best of 5 runs before and after the changes are: parsed 1709399 lines (3763 files) in 1.707402159s (1001169 lines/s) allocated 449.282Mb (263.137Mb/s) parsed 1709329 lines (3765 files) in 1.706663154s (1001562 lines/s) allocated 449.290Mb (263.256Mb/s) Change-Id: Idcc4f83393fcadd6579700e3602c09496ea2625b Reviewed-on: https://go-review.googlesource.com/95357 Reviewed-by: Robert Griesemer --- diff --git a/src/cmd/compile/fmt_test.go b/src/cmd/compile/fmt_test.go index ff82378385..f3f66a66ae 100644 --- a/src/cmd/compile/fmt_test.go +++ b/src/cmd/compile/fmt_test.go @@ -649,10 +649,8 @@ var knownFormats = map[string]string{ "cmd/compile/internal/ssa.register %d": "", "cmd/compile/internal/syntax.Expr %#v": "", "cmd/compile/internal/syntax.Node %T": "", - "cmd/compile/internal/syntax.Operator %d": "", "cmd/compile/internal/syntax.Operator %s": "", "cmd/compile/internal/syntax.position %s": "", - "cmd/compile/internal/syntax.token %d": "", "cmd/compile/internal/syntax.token %q": "", "cmd/compile/internal/syntax.token %s": "", "cmd/compile/internal/types.EType %d": "", diff --git a/src/cmd/compile/internal/syntax/operator_string.go b/src/cmd/compile/internal/syntax/operator_string.go new file mode 100644 index 0000000000..3c759b2e9b --- /dev/null +++ b/src/cmd/compile/internal/syntax/operator_string.go @@ -0,0 +1,17 @@ +// Code generated by "stringer -type Operator -linecomment"; DO NOT EDIT. + +package syntax + +import "strconv" + +const _Operator_name = ":!<-||&&==!=<<=>>=+-|^*/%&&^<<>>" + +var _Operator_index = [...]uint8{0, 1, 2, 4, 6, 8, 10, 12, 13, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 28, 30, 32} + +func (i Operator) String() string { + i -= 1 + if i >= Operator(len(_Operator_index)-1) { + return "Operator(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _Operator_name[_Operator_index[i]:_Operator_index[i+1]] +} diff --git a/src/cmd/compile/internal/syntax/scanner.go b/src/cmd/compile/internal/syntax/scanner.go index dbb2387f8f..7db33fb6b9 100644 --- a/src/cmd/compile/internal/syntax/scanner.go +++ b/src/cmd/compile/internal/syntax/scanner.go @@ -348,7 +348,7 @@ func (s *scanner) ident() { // possibly a keyword if len(lit) >= 2 { - if tok := keywordMap[hash(lit)]; tok != 0 && tokstrings[tok] == string(lit) { + if tok := keywordMap[hash(lit)]; tok != 0 && tokStrFast(tok) == string(lit) { s.nlsemi = contains(1<<_Break|1<<_Continue|1<<_Fallthrough|1<<_Return, tok) s.tok = tok return @@ -360,6 +360,12 @@ func (s *scanner) ident() { s.tok = _Name } +// tokStrFast is a faster version of token.String, which assumes that tok +// is one of the valid tokens - and can thus skip bounds checks. +func tokStrFast(tok token) string { + return _token_name[_token_index[tok-1]:_token_index[tok]] +} + func (s *scanner) isIdentRune(c rune, first bool) bool { switch { case unicode.IsLetter(c) || c == '_': @@ -387,7 +393,7 @@ var keywordMap [1 << 6]token // size must be power of two func init() { // populate keywordMap for tok := _Break; tok <= _Var; tok++ { - h := hash([]byte(tokstrings[tok])) + h := hash([]byte(tok.String())) if keywordMap[h] != 0 { panic("imperfect hash") } diff --git a/src/cmd/compile/internal/syntax/token_string.go b/src/cmd/compile/internal/syntax/token_string.go new file mode 100644 index 0000000000..3cf5473feb --- /dev/null +++ b/src/cmd/compile/internal/syntax/token_string.go @@ -0,0 +1,17 @@ +// Code generated by "stringer -type token -linecomment"; DO NOT EDIT. + +package syntax + +import "strconv" + +const _token_name = "EOFnameliteralopop=opop=:=<-*([{)]},;:....breakcasechanconstcontinuedefaultdeferelsefallthroughforfuncgogotoifimportinterfacemappackagerangereturnselectstructswitchtypevar" + +var _token_index = [...]uint8{0, 3, 7, 14, 16, 19, 23, 24, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 42, 47, 51, 55, 60, 68, 75, 80, 84, 95, 98, 102, 104, 108, 110, 116, 125, 128, 135, 140, 146, 152, 158, 164, 168, 171, 171} + +func (i token) String() string { + i -= 1 + if i >= token(len(_token_index)-1) { + return "token(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _token_name[_token_index[i]:_token_index[i+1]] +} diff --git a/src/cmd/compile/internal/syntax/tokens.go b/src/cmd/compile/internal/syntax/tokens.go index e49a027029..e00255a45e 100644 --- a/src/cmd/compile/internal/syntax/tokens.go +++ b/src/cmd/compile/internal/syntax/tokens.go @@ -4,68 +4,70 @@ package syntax -import "fmt" - type token uint +//go:generate stringer -type token -linecomment + const ( - _ token = iota - _EOF + _ token = iota + _EOF // EOF // names and literals - _Name - _Literal + _Name // name + _Literal // literal // operators and operations - _Operator // excluding '*' (_Star) - _AssignOp - _IncOp - _Assign - _Define - _Arrow - _Star + // _Operator is excluding '*' (_Star) + _Operator // op + _AssignOp // op= + _IncOp // opop + _Assign // = + _Define // := + _Arrow // <- + _Star // * // delimiters - _Lparen - _Lbrack - _Lbrace - _Rparen - _Rbrack - _Rbrace - _Comma - _Semi - _Colon - _Dot - _DotDotDot + _Lparen // ( + _Lbrack // [ + _Lbrace // { + _Rparen // ) + _Rbrack // ] + _Rbrace // } + _Comma // , + _Semi // ; + _Colon // : + _Dot // . + _DotDotDot // ... // keywords - _Break - _Case - _Chan - _Const - _Continue - _Default - _Defer - _Else - _Fallthrough - _For - _Func - _Go - _Goto - _If - _Import - _Interface - _Map - _Package - _Range - _Return - _Select - _Struct - _Switch - _Type - _Var - - tokenCount + _Break // break + _Case // case + _Chan // chan + _Const // const + _Continue // continue + _Default // default + _Defer // defer + _Else // else + _Fallthrough // fallthrough + _For // for + _Func // func + _Go // go + _Goto // goto + _If // if + _Import // import + _Interface // interface + _Map // map + _Package // package + _Range // range + _Return // return + _Select // select + _Struct // struct + _Switch // switch + _Type // type + _Var // var + + // empty line comment to exclude it from .String + tokenCount // ) const ( @@ -80,75 +82,6 @@ const ( Defer = _Defer ) -var tokstrings = [...]string{ - // source control - _EOF: "EOF", - - // names and literals - _Name: "name", - _Literal: "literal", - - // operators and operations - _Operator: "op", - _AssignOp: "op=", - _IncOp: "opop", - _Assign: "=", - _Define: ":=", - _Arrow: "<-", - _Star: "*", - - // delimiters - _Lparen: "(", - _Lbrack: "[", - _Lbrace: "{", - _Rparen: ")", - _Rbrack: "]", - _Rbrace: "}", - _Comma: ",", - _Semi: ";", - _Colon: ":", - _Dot: ".", - _DotDotDot: "...", - - // keywords - _Break: "break", - _Case: "case", - _Chan: "chan", - _Const: "const", - _Continue: "continue", - _Default: "default", - _Defer: "defer", - _Else: "else", - _Fallthrough: "fallthrough", - _For: "for", - _Func: "func", - _Go: "go", - _Goto: "goto", - _If: "if", - _Import: "import", - _Interface: "interface", - _Map: "map", - _Package: "package", - _Range: "range", - _Return: "return", - _Select: "select", - _Struct: "struct", - _Switch: "switch", - _Type: "type", - _Var: "var", -} - -func (tok token) String() string { - var s string - if 0 <= tok && int(tok) < len(tokstrings) { - s = tokstrings[tok] - } - if s == "" { - s = fmt.Sprintf("", tok) - } - return s -} - // Make sure we have at most 64 tokens so we can use them in a set. const _ uint64 = 1 << (tokenCount - 1) @@ -169,11 +102,15 @@ const ( type Operator uint +//go:generate stringer -type Operator -linecomment + const ( - _ Operator = iota - Def // := - Not // ! - Recv // <- + _ Operator = iota + + // Def is the : in := + Def // : + Not // ! + Recv // <- // precOrOr OrOr // || @@ -205,53 +142,6 @@ const ( Shr // >> ) -var opstrings = [...]string{ - // prec == 0 - Def: ":", // : in := - Not: "!", - Recv: "<-", - - // precOrOr - OrOr: "||", - - // precAndAnd - AndAnd: "&&", - - // precCmp - Eql: "==", - Neq: "!=", - Lss: "<", - Leq: "<=", - Gtr: ">", - Geq: ">=", - - // precAdd - Add: "+", - Sub: "-", - Or: "|", - Xor: "^", - - // precMul - Mul: "*", - Div: "/", - Rem: "%", - And: "&", - AndNot: "&^", - Shl: "<<", - Shr: ">>", -} - -func (op Operator) String() string { - var s string - if 0 <= op && int(op) < len(opstrings) { - s = opstrings[op] - } - if s == "" { - s = fmt.Sprintf("", op) - } - return s -} - // Operator precedences const ( _ = iota