From: Matthew Dempsky Date: Tue, 30 Aug 2016 23:31:53 +0000 (-0700) Subject: cmd/compile: handle pragmas immediately with -newparser=1 X-Git-Tag: go1.8beta1~1538 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=ee161e859166b8b8b077998c0101f56c18b18329;p=gostls13.git cmd/compile: handle pragmas immediately with -newparser=1 Instead of saving all pragmas and processing them after parsing is finished, process them immediately during scanning like the current lexer does. This is a bit unfortunate because it means we can't use syntax.ParseFile to concurrently parse files yet, but it fixes how we report syntax errors in the presence of //line pragmas. While here, add a bunch more gcCompat entries to syntax/parser.go to get "go build -toolexec='toolstash -cmp' std cmd" passing. There are still a few remaining cases only triggered building unit tests, but this seems like a nice checkpoint. Change-Id: Iaf3bbcf2849857a460496f31eea228e0c585ce13 Reviewed-on: https://go-review.googlesource.com/28226 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Robert Griesemer --- diff --git a/src/cmd/compile/internal/gc/lex.go b/src/cmd/compile/internal/gc/lex.go index 309da90458..5d3da58010 100644 --- a/src/cmd/compile/internal/gc/lex.go +++ b/src/cmd/compile/internal/gc/lex.go @@ -7,6 +7,7 @@ package gc import ( "bufio" "bytes" + "cmd/compile/internal/syntax" "cmd/internal/obj" "fmt" "io" @@ -60,7 +61,7 @@ func plan9quote(s string) string { return s } -type Pragma uint16 +type Pragma syntax.Pragma const ( Nointerface Pragma = 1 << iota diff --git a/src/cmd/compile/internal/gc/noder.go b/src/cmd/compile/internal/gc/noder.go index 8e2b9ef5fc..a945038c7a 100644 --- a/src/cmd/compile/internal/gc/noder.go +++ b/src/cmd/compile/internal/gc/noder.go @@ -6,7 +6,6 @@ package gc import ( "fmt" - "sort" "strconv" "strings" "unicode/utf8" @@ -15,23 +14,19 @@ import ( ) func parseFile(filename string) { - errh := func(_, line int, msg string) { - yyerrorl(lexlineno+int32(line)-1, "%s", msg) - } - - file, err := syntax.ReadFile(filename, errh, 0) + p := noder{baseline: lexlineno} + file, err := syntax.ReadFile(filename, p.error, p.pragma, 0) if err != nil { Fatalf("syntax.ReadFile %s: %v", filename, err) } - p := noder{pragmas: file.Pragmas} - - p.lineno(file.PkgName) - mkpackage(file.PkgName.Value) + p.file(file) - xtop = append(xtop, p.decls(file.DeclList, true)...) - p.globalPragmas() - lexlineno += p.maxline + if !imported_unsafe { + for _, x := range p.linknames { + p.error(0, x, "//go:linkname only allowed in Go files that import \"unsafe\"") + } + } if nsyntaxerrors == 0 { testdclstack() @@ -40,13 +35,21 @@ func parseFile(filename string) { // noder transforms package syntax's AST into a Nod tree. type noder struct { - indent []byte - pragmas []syntax.Pragma - pline int32 - maxline int32 + baseline int32 + linknames []int // tracks //go:linkname lines } -func (p *noder) decls(decls []syntax.Decl, top bool) (l []*Node) { +func (p *noder) file(file *syntax.File) { + p.lineno(file.PkgName) + mkpackage(file.PkgName.Value) + + xtop = append(xtop, p.decls(file.DeclList)...) + + lexlineno = p.baseline + int32(file.Lines) - 1 + lineno = lexlineno +} + +func (p *noder) decls(decls []syntax.Decl) (l []*Node) { var lastConstGroup *syntax.Group var lastConstRHS []*Node var iotaVal int32 @@ -59,6 +62,7 @@ func (p *noder) decls(decls []syntax.Decl, top bool) (l []*Node) { case *syntax.VarDecl: l = append(l, p.varDecl(decl)...) + case *syntax.ConstDecl: // Tricky to handle golang.org/issue/15550 correctly. @@ -91,10 +95,6 @@ func (p *noder) decls(decls []syntax.Decl, top bool) (l []*Node) { default: panic("unhandled Decl") } - - if top { - p.pline = p.maxline - } } return @@ -102,7 +102,7 @@ func (p *noder) decls(decls []syntax.Decl, top bool) (l []*Node) { func (p *noder) importDecl(imp *syntax.ImportDecl) { val := p.basicLit(imp.Path) - importfile(&val, p.indent) + importfile(&val, nil) ipkg := importpkg importpkg = nil @@ -159,6 +159,7 @@ func (p *noder) varDecl(decl *syntax.VarDecl) []*Node { exprs = p.exprList(decl.Values) } + p.lineno(decl) return variter(names, typ, exprs) } @@ -216,7 +217,7 @@ func (p *noder) funcDecl(fun *syntax.FuncDecl) *Node { } } - pragma := p.pragma() + pragma := Pragma(fun.Pragma) f.Nbody.Set(body) f.Noescape = pragma&Noescape != 0 @@ -224,7 +225,7 @@ func (p *noder) funcDecl(fun *syntax.FuncDecl) *Node { Yyerror("can only use //go:noescape with external func implementations") } f.Func.Pragma = pragma - lineno = lexlineno + int32(fun.EndLine) - 1 + lineno = p.baseline + int32(fun.EndLine) - 1 f.Func.Endlineno = lineno funcbody(f) @@ -345,15 +346,19 @@ func (p *noder) expr(expr syntax.Expr) *Node { if expr.Type != nil { n.Right = p.expr(expr.Type) } - // TODO(mdempsky): Should apply wrapname to n.List nodes. - n.List.Set(p.exprs(expr.ElemList)) + l := p.exprs(expr.ElemList) + for i, e := range l { + l[i] = p.wrapname(expr.ElemList[i], e) + } + n.List.Set(l) + lineno = p.baseline + int32(expr.EndLine) - 1 return n case *syntax.KeyValueExpr: return p.nod(expr, OKEY, p.expr(expr.Key), p.wrapname(expr.Value, p.expr(expr.Value))) case *syntax.FuncLit: closurehdr(p.typeExpr(expr.Type)) body := p.stmts(expr.Body) - lineno = lexlineno + int32(expr.EndLine) - 1 + lineno = p.baseline + int32(expr.EndLine) - 1 return p.setlineno(expr, closurebody(body)) case *syntax.ParenExpr: return p.nod(expr, OPAREN, p.expr(expr.X), nil) @@ -398,7 +403,9 @@ func (p *noder) expr(expr syntax.Expr) *Node { x = unparen(x) // TODO(mdempsky): Needed? if x.Op == OCOMPLIT { // Special case for &T{...}: turn into (*T){...}. - x.Right = p.nod(expr, OIND, x.Right, nil) + // TODO(mdempsky): Switch back to p.nod after we + // get rid of gcCompat. + x.Right = Nod(OIND, x.Right, nil) x.Right.Implicit = true return x } @@ -577,7 +584,7 @@ func (p *noder) stmt(stmt syntax.Stmt) *Node { case *syntax.SendStmt: return p.nod(stmt, OSEND, p.expr(stmt.Chan), p.expr(stmt.Value)) case *syntax.DeclStmt: - return liststmt(p.decls(stmt.DeclList, false)) + return liststmt(p.decls(stmt.DeclList)) case *syntax.AssignStmt: if stmt.Op != 0 && stmt.Op != syntax.Def { n := p.nod(stmt, OASOP, p.expr(stmt.Lhs), p.expr(stmt.Rhs)) @@ -975,10 +982,7 @@ func (p *noder) setlineno(src syntax.Node, dst *Node) *Node { // TODO(mdempsky): Shouldn't happen. Fix package syntax. return dst } - if l > p.maxline { - p.maxline = l - } - dst.Lineno = lexlineno + l - 1 + dst.Lineno = p.baseline + l - 1 return dst } @@ -991,78 +995,57 @@ func (p *noder) lineno(n syntax.Node) { // TODO(mdempsky): Shouldn't happen. Fix package syntax. return } - if l > p.maxline { - p.maxline = l - } - lineno = lexlineno + l - 1 + lineno = p.baseline + l - 1 } -func (p *noder) pragma() Pragma { - lo := sort.Search(len(p.pragmas), func(i int) bool { return int32(p.pragmas[i].Line) >= p.pline }) - hi := sort.Search(len(p.pragmas), func(i int) bool { return int32(p.pragmas[i].Line) > p.maxline }) +func (p *noder) error(_, line int, msg string) { + yyerrorl(p.baseline+int32(line)-1, "%s", msg) +} - var res Pragma - for _, prag := range p.pragmas[lo:hi] { - text := prag.Text - if !strings.HasPrefix(text, "go:") { - continue +func (p *noder) pragma(pos, line int, text string) syntax.Pragma { + switch { + case strings.HasPrefix(text, "line "): + i := strings.IndexByte(text, ':') + if i < 0 { + break + } + n, err := strconv.Atoi(text[i+1:]) + if err != nil { + // todo: make this an error instead? it is almost certainly a bug. + break } + if n > 1e8 { + p.error(pos, line, "line number out of range") + errorexit() + } + if n <= 0 { + break + } + lexlineno = p.baseline + int32(line) + linehistupdate(text[5:i], n) + + case strings.HasPrefix(text, "go:linkname "): + // Record line number so we can emit an error later if + // the file doesn't import package unsafe. + p.linknames = append(p.linknames, line) + + f := strings.Fields(text) + if len(f) != 3 { + p.error(pos, line, "usage: //go:linkname localname linkname") + break + } + Lookup(f[1]).Linkname = f[2] + case strings.HasPrefix(text, "go:cgo_"): + pragcgobuf += pragcgo(text) + fallthrough // because of //go:cgo_unsafe_args + default: verb := text if i := strings.Index(text, " "); i >= 0 { verb = verb[:i] } - - res |= PragmaValue(verb) + return syntax.Pragma(PragmaValue(verb)) } - return res -} - -func (p *noder) globalPragmas() { - origlexlineno := lexlineno - defer func() { - lexlineno = origlexlineno - }() - - for _, prag := range p.pragmas { - text := prag.Text - - if strings.HasPrefix(text, "go:cgo_") { - pragcgobuf += pragcgo(text) - } - if strings.HasPrefix(text, "go:linkname ") { - if !imported_unsafe { - Yyerror("//go:linkname only allowed in Go files that import \"unsafe\"") - } - f := strings.Fields(text) - if len(f) != 3 { - Yyerror("usage: //go:linkname localname linkname") - break - } - Lookup(f[1]).Linkname = f[2] - } - - // TODO(mdempsky): Move into package syntax. - if strings.HasPrefix(text, "line ") { - i := strings.IndexByte(text, ':') - if i < 0 { - continue - } - n, err := strconv.Atoi(text[i+1:]) - if err != nil { - // todo: make this an error instead? it is almost certainly a bug. - continue - } - if n > 1e8 { - Yyerror("line number out of range") - errorexit() - } - if n <= 0 { - continue - } - lexlineno = origlexlineno + int32(prag.Line) - linehistupdate(text[5:i], n) - } - } + return 0 } diff --git a/src/cmd/compile/internal/syntax/dumper_test.go b/src/cmd/compile/internal/syntax/dumper_test.go index fd38e7ca78..1ee1d982d0 100644 --- a/src/cmd/compile/internal/syntax/dumper_test.go +++ b/src/cmd/compile/internal/syntax/dumper_test.go @@ -14,7 +14,7 @@ func TestDump(t *testing.T) { t.Skip("skipping test in short mode") } - ast, err := ReadFile(*src, nil, 0) + ast, err := ReadFile(*src, nil, nil, 0) if err != nil { t.Fatal(err) } diff --git a/src/cmd/compile/internal/syntax/nodes.go b/src/cmd/compile/internal/syntax/nodes.go index e56b1235fe..280a2e8f69 100644 --- a/src/cmd/compile/internal/syntax/nodes.go +++ b/src/cmd/compile/internal/syntax/nodes.go @@ -10,6 +10,7 @@ package syntax type Node interface { Line() uint32 aNode() + init(p *parser) } type node struct { @@ -35,16 +36,10 @@ func (n *node) init(p *parser) { type File struct { PkgName *Name DeclList []Decl - Pragmas []Pragma Lines int node } -type Pragma struct { - Line int - Text string -} - // ---------------------------------------------------------------------------- // Declarations @@ -90,6 +85,7 @@ type ( Name *Name Type *FuncType Body []Stmt // nil means no body (forward declaration) + Pragma Pragma // TODO(mdempsky): Cleaner solution. EndLine uint32 // TODO(mdempsky): Cleaner solution. decl } @@ -130,7 +126,8 @@ type ( CompositeLit struct { Type Expr // nil means no literal type ElemList []Expr - NKeys int // number of elements with keys + NKeys int // number of elements with keys + EndLine uint32 // TODO(mdempsky): Cleaner solution. expr } diff --git a/src/cmd/compile/internal/syntax/parser.go b/src/cmd/compile/internal/syntax/parser.go index 9544001a2e..ae9c14c811 100644 --- a/src/cmd/compile/internal/syntax/parser.go +++ b/src/cmd/compile/internal/syntax/parser.go @@ -28,7 +28,7 @@ type parser struct { nerrors int // error count } -func (p *parser) init(src io.Reader, errh ErrorHandler) { +func (p *parser) init(src io.Reader, errh ErrorHandler, pragh PragmaHandler) { p.scanner.init(src, func(pos, line int, msg string) { p.nerrors++ if !debug && errh != nil { @@ -36,7 +36,7 @@ func (p *parser) init(src io.Reader, errh ErrorHandler) { return } panic(fmt.Sprintf("%d: %s\n", line, msg)) - }) + }, pragh) p.fnest = 0 p.xnest = 0 @@ -245,6 +245,10 @@ func (p *parser) file() *File { continue } + // Reset p.pragma BEFORE advancing to the next token (consuming ';') + // since comments before may set pragmas for the next function decl. + p.pragma = 0 + if p.tok != _EOF && !p.got(_Semi) { p.syntax_error("after top level declaration") p.advance(_Const, _Type, _Var, _Func) @@ -253,7 +257,6 @@ func (p *parser) file() *File { // p.tok == _EOF f.Lines = p.source.line - f.Pragmas = p.pragmas return f } @@ -372,6 +375,9 @@ func (p *parser) varDecl(group *Group) Decl { } } d.Group = group + if gcCompat { + d.init(p) + } return d } @@ -426,8 +432,12 @@ func (p *parser) funcDecl() *FuncDecl { f.Name = p.name() f.Type = p.funcType() + if gcCompat { + f.node = f.Type.node + } f.Body = p.funcBody() + f.Pragma = p.pragma f.EndLine = uint32(p.line) // TODO(gri) deal with function properties @@ -465,6 +475,9 @@ func (p *parser) binaryExpr(prec int) Expr { tprec := p.prec p.next() t.Y = p.binaryExpr(tprec) + if gcCompat { + t.init(p) + } x = t } return x @@ -485,6 +498,9 @@ func (p *parser) unaryExpr() Expr { x.Op = p.op p.next() x.X = p.unaryExpr() + if gcCompat { + x.init(p) + } return x case And: @@ -730,6 +746,9 @@ loop: p.syntax_error("expecting name or (") p.advance(_Semi, _Rparen) } + if gcCompat { + x.init(p) + } case _Lbrack: p.next() @@ -851,6 +870,9 @@ func (p *parser) complitexpr() *CompositeLit { l.init(p) l.Key = e l.Value = p.bare_complitexpr() + if gcCompat { + l.init(p) + } e = l x.NKeys++ } @@ -860,6 +882,7 @@ func (p *parser) complitexpr() *CompositeLit { } } + x.EndLine = uint32(p.line) p.xnest-- p.want(_Rbrace) @@ -996,6 +1019,9 @@ func (p *parser) funcType() *FuncType { typ.init(p) typ.ParamList = p.paramList() typ.ResultList = p.funcResult() + if gcCompat { + typ.init(p) + } return typ } @@ -1134,6 +1160,10 @@ func (p *parser) addField(styp *StructType, name *Name, typ Expr, tag *BasicLit) f.Type = typ styp.FieldList = append(styp.FieldList, f) + if gcCompat && name != nil { + f.node = name.node + } + if debug && tag != nil && len(styp.FieldList) != len(styp.TagList) { panic("inconsistent struct field list") } @@ -1443,6 +1473,9 @@ func (p *parser) simpleStmt(lhs Expr, rangeOk bool) SimpleStmt { s.init(p) s.Chan = lhs s.Value = p.expr() + if gcCompat { + s.init(p) + } return s default: @@ -1509,6 +1542,9 @@ func (p *parser) rangeClause(lhs Expr, def bool) *RangeClause { r.Lhs = lhs r.Def = def r.X = p.expr() + if gcCompat { + r.init(p) + } return r } @@ -1583,6 +1619,9 @@ func (p *parser) forStmt() Stmt { p.want(_For) s.Init, s.Cond, s.Post = p.header(true) + if gcCompat { + s.init(p) + } s.Body = p.stmtBody("for clause") return s @@ -1672,6 +1711,10 @@ func (p *parser) ifStmt() *IfStmt { p.error("missing condition in if statement") } + if gcCompat { + s.init(p) + } + s.Then = p.stmtBody("if clause") if p.got(_Else) { @@ -1914,6 +1957,9 @@ func (p *parser) stmt() Stmt { if p.tok != _Semi && p.tok != _Rbrace { s.Results = p.exprList() } + if gcCompat { + s.init(p) + } return s case _Semi: diff --git a/src/cmd/compile/internal/syntax/parser_test.go b/src/cmd/compile/internal/syntax/parser_test.go index 12fc019414..8e6b77d0c6 100644 --- a/src/cmd/compile/internal/syntax/parser_test.go +++ b/src/cmd/compile/internal/syntax/parser_test.go @@ -22,7 +22,7 @@ var src = flag.String("src", "parser.go", "source file to parse") var verify = flag.Bool("verify", false, "verify idempotent printing") func TestParse(t *testing.T) { - _, err := ReadFile(*src, nil, 0) + _, err := ReadFile(*src, nil, nil, 0) if err != nil { t.Fatal(err) } @@ -52,7 +52,7 @@ func TestStdLib(t *testing.T) { if debug { fmt.Printf("parsing %s\n", filename) } - ast, err := ReadFile(filename, nil, 0) + ast, err := ReadFile(filename, nil, nil, 0) if err != nil { t.Fatal(err) } @@ -133,7 +133,7 @@ func verifyPrint(filename string, ast1 *File) { panic(err) } - ast2, err := ReadBytes(buf1.Bytes(), nil, 0) + ast2, err := ReadBytes(buf1.Bytes(), nil, nil, 0) if err != nil { panic(err) } diff --git a/src/cmd/compile/internal/syntax/printer_test.go b/src/cmd/compile/internal/syntax/printer_test.go index e8c2201e60..a2d43068dd 100644 --- a/src/cmd/compile/internal/syntax/printer_test.go +++ b/src/cmd/compile/internal/syntax/printer_test.go @@ -15,7 +15,7 @@ func TestPrint(t *testing.T) { t.Skip("skipping test in short mode") } - ast, err := ReadFile(*src, nil, 0) + ast, err := ReadFile(*src, nil, nil, 0) if err != nil { t.Fatal(err) } diff --git a/src/cmd/compile/internal/syntax/scanner.go b/src/cmd/compile/internal/syntax/scanner.go index d02bb6d11b..e78950ad1a 100644 --- a/src/cmd/compile/internal/syntax/scanner.go +++ b/src/cmd/compile/internal/syntax/scanner.go @@ -15,6 +15,7 @@ import ( type scanner struct { source nlsemi bool // if set '\n' and EOF translate to ';' + pragma Pragma // current token, valid after calling next() pos, line int @@ -24,12 +25,13 @@ type scanner struct { op Operator // valid if tok is _Operator, _AssignOp, or _IncOp prec int // valid if tok is _Operator, _AssignOp, or _IncOp - pragmas []Pragma + pragh PragmaHandler } -func (s *scanner) init(src io.Reader, errh ErrorHandler) { +func (s *scanner) init(src io.Reader, errh ErrorHandler, pragh PragmaHandler) { s.source.init(src, errh) s.nlsemi = false + s.pragh = pragh } func (s *scanner) next() { @@ -540,6 +542,10 @@ func (s *scanner) lineComment() { // recognize pragmas var prefix string r := s.getr() + if s.pragh == nil { + goto skip + } + switch r { case 'g': prefix = "go:" @@ -565,10 +571,7 @@ func (s *scanner) lineComment() { } r = s.getr() } - s.pragmas = append(s.pragmas, Pragma{ - Line: s.line, - Text: strings.TrimSuffix(string(s.stopLit()), "\r"), - }) + s.pragma |= s.pragh(0, s.line, strings.TrimSuffix(string(s.stopLit()), "\r")) return skip: diff --git a/src/cmd/compile/internal/syntax/scanner_test.go b/src/cmd/compile/internal/syntax/scanner_test.go index 70b986ed4e..38a7e0da4c 100644 --- a/src/cmd/compile/internal/syntax/scanner_test.go +++ b/src/cmd/compile/internal/syntax/scanner_test.go @@ -22,7 +22,7 @@ func TestScanner(t *testing.T) { defer src.Close() var s scanner - s.init(src, nil) + s.init(src, nil, nil) for { s.next() if s.tok == _EOF { @@ -51,7 +51,7 @@ func TestTokens(t *testing.T) { // scan source var got scanner - got.init(&bytesReader{buf}, nil) + got.init(&bytesReader{buf}, nil, nil) got.next() for i, want := range sampleTokens { nlsemi := false @@ -338,7 +338,7 @@ func TestScanErrors(t *testing.T) { } else if nerrors > 1 { t.Errorf("%q: got unexpected %q at pos = %d, line = %d", test.src, msg, pos, line) } - }) + }, nil) for { s.next() diff --git a/src/cmd/compile/internal/syntax/syntax.go b/src/cmd/compile/internal/syntax/syntax.go index 85dddaa47f..6c0abd118d 100644 --- a/src/cmd/compile/internal/syntax/syntax.go +++ b/src/cmd/compile/internal/syntax/syntax.go @@ -12,17 +12,27 @@ import ( type Mode uint +// A Pragma value is a set of flags that augment a function +// declaration. Callers may assign meaning to the flags as +// appropriate. +type Pragma uint16 + type ErrorHandler func(pos, line int, msg string) +// A PragmaHandler is used to process //line and //go: directives as +// they're scanned. The returned Pragma value will be unioned into the +// next FuncDecl node. +type PragmaHandler func(pos, line int, text string) Pragma + // TODO(gri) These need a lot more work. -func ReadFile(filename string, errh ErrorHandler, mode Mode) (*File, error) { +func ReadFile(filename string, errh ErrorHandler, pragh PragmaHandler, mode Mode) (*File, error) { src, err := os.Open(filename) if err != nil { return nil, err } defer src.Close() - return Read(src, errh, mode) + return Read(src, errh, pragh, mode) } type bytesReader struct { @@ -38,13 +48,13 @@ func (r *bytesReader) Read(p []byte) (int, error) { return 0, io.EOF } -func ReadBytes(src []byte, errh ErrorHandler, mode Mode) (*File, error) { - return Read(&bytesReader{src}, errh, mode) +func ReadBytes(src []byte, errh ErrorHandler, pragh PragmaHandler, mode Mode) (*File, error) { + return Read(&bytesReader{src}, errh, pragh, mode) } -func Read(src io.Reader, errh ErrorHandler, mode Mode) (*File, error) { +func Read(src io.Reader, errh ErrorHandler, pragh PragmaHandler, mode Mode) (*File, error) { var p parser - p.init(src, errh) + p.init(src, errh, pragh) p.next() ast := p.file()