{.section PAst}
<pre>
- {@|html}
+ {@ FSet|html}
</pre>
{.end}
{.section PDoc}
<!-- PackageName is printed as title by the top-level template -->
{.section IsPkg}
+ {# ImportPath is a string - no need for FSet}
<p><code>import "{ImportPath|html-esc}"</code></p>
{.end}
{Doc|html-comment}
{.section IsPkg}
{.section Filenames}
+ {# Filenames are strings - no need for FSet}
<p>
<h4>Package files</h4>
<span style="font-size:90%">
<h2 id="Constants">Constants</h2>
{.repeated section @}
{Doc|html-comment}
- <pre>{Decl|html}</pre>
+ <pre>{Decl FSet|html}</pre>
{.end}
{.end}
{.section Vars}
<h2 id="Variables">Variables</h2>
{.repeated section @}
{Doc|html-comment}
- <pre>{Decl|html}</pre>
+ <pre>{Decl FSet|html}</pre>
{.end}
{.end}
{.section Funcs}
{.repeated section @}
- <h2 id="{Name|html-esc}">func <a href="/{Decl|url-pos}">{Name|html-esc}</a></h2>
- <p><code>{Decl|html}</code></p>
+ {# Name is a string - no need for FSet}
+ <h2 id="{Name|html-esc}">func <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h2>
+ <p><code>{Decl FSet|html}</code></p>
{Doc|html-comment}
{.end}
{.end}
{.section Types}
{.repeated section @}
- <h2 id="{Type.Name|html-esc}">type <a href="/{Decl|url-pos}">{Type.Name|html-esc}</a></h2>
+ {# Type.Name is a string - no need for FSet}
+ <h2 id="{Type.Name FSet|html-esc}">type <a href="/{Decl FSet|url-pos}">{Type.Name FSet|html-esc}</a></h2>
{Doc|html-comment}
- <p><pre>{Decl|html}</pre></p>
+ <p><pre>{Decl FSet|html}</pre></p>
{.repeated section Consts}
{Doc|html-comment}
- <pre>{Decl|html}</pre>
+ <pre>{Decl FSet|html}</pre>
{.end}
{.repeated section Vars}
{Doc|html-comment}
- <pre>{Decl|html}</pre>
+ <pre>{Decl FSet|html}</pre>
{.end}
{.repeated section Factories}
- <h3 id="{Type.Name|html-esc}.{Name|html-esc}">func <a href="/{Decl|url-pos}">{Name|html-esc}</a></h3>
- <p><code>{Decl|html}</code></p>
+ <h3 id="{Type.Name FSet|html-esc}.{Name|html-esc}">func <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h3>
+ <p><code>{Decl FSet|html}</code></p>
{Doc|html-comment}
{.end}
{.repeated section Methods}
- <h3 id="{Type.Name|html-esc}.{Name|html-esc}">func ({Recv|html}) <a href="/{Decl|url-pos}">{Name|html-esc}</a></h3>
- <p><code>{Decl|html}</code></p>
+ <h3 id="{Type.Name FSet|html-esc}.{Name|html-esc}">func ({Recv FSet|html}) <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h3>
+ <p><code>{Decl FSet|html}</code></p>
{Doc|html-comment}
{.end}
{.end}
{.section PList}
<h2>Other packages</h2>
<p>
+ {# PLIst entries are strings - no need for FSet}
{.repeated section @}
- <a href="?p={@|html}">{@|html}</a><br />
+ <a href="?p={@|html-esc}">{@|html-esc}</a><br />
{.end}
</p>
{.end}
{.section Dirs}
+ {# DirList entries are numbers and strings - no need for FSet}
<h2 id="Subdirectories">Subdirectories</h2>
<p>
<table class="layout">
CONSTANTS
{.repeated section @}
-{Decl}
+{Decl FSet}
{Doc}
{.end}
{.end}
VARIABLES
{.repeated section @}
-{Decl}
+{Decl FSet}
{Doc}
{.end}
{.end}
FUNCTIONS
{.repeated section @}
-{Decl}
+{Decl FSet}
{Doc}
{.end}
{.end}
TYPES
{.repeated section @}
-{Decl}
+{Decl FSet}
{Doc}
{.repeated section Consts}
-{Decl}
+{Decl FSet}
{Doc}
{.end}
{.repeated section Vars}
-{Decl}
+{Decl FSet}
{Doc}
{.end}
{.repeated section Factories}
-{Decl}
+{Decl FSet}
{Doc}
{.end}
{.repeated section Methods}
-{Decl}
+{Decl FSet}
{Doc}
{.end}
{.end}
"go/doc"
"go/parser"
"go/scanner"
+ "go/token"
"os"
"strings"
)
func parse(name string, flags uint) *ast.File {
- ast1, err := parser.ParseFile(name, nil, flags)
+ ast1, err := parser.ParseFile(fset, name, nil, flags)
if err != nil {
if list, ok := err.(scanner.ErrorList); ok {
// If err is a scanner.ErrorList, its String will print just
}
}
if !sawC {
- error(noPos, `cannot find import "C"`)
+ error(token.NoPos, `cannot find import "C"`)
}
// In ast2, strip the import "C" line.
// everything else just recurs
default:
- error(noPos, "unexpected type %T in walk", x, visit)
+ error(token.NoPos, "unexpected type %T in walk", x, visit)
panic("unexpected type")
case nil:
if _, err := strconv.Atoi(n.Define); err == nil {
ok = true
} else if n.Define[0] == '"' || n.Define[0] == '\'' {
- _, err := parser.ParseExpr("", n.Define)
+ _, err := parser.ParseExpr(fset, "", n.Define)
if err == nil {
ok = true
}
case strings.Contains(line, ": statement with no effect"):
what = "not-type" // const or func or var
case strings.Contains(line, "undeclared"):
- error(noPos, "%s", strings.TrimSpace(line[colon+1:]))
+ error(token.NoPos, "%s", strings.TrimSpace(line[colon+1:]))
case strings.Contains(line, "is not an integer constant"):
isConst[i] = false
continue
if n.Kind != "" {
continue
}
- error(noPos, "could not determine kind of name for C.%s", n.Go)
+ error(token.NoPos, "could not determine kind of name for C.%s", n.Go)
}
if nerrors > 0 {
fatal("unresolved names")
Context string // "type", "expr", "call", or "call2"
}
-func (r *Ref) Pos() token.Position {
+func (r *Ref) Pos() token.Pos {
return (*r.Expr).Pos()
}
"arm": 4,
}
+var fset = token.NewFileSet()
+
func main() {
flag.Usage = usage
flag.Parse()
if p.PackageName == "" {
p.PackageName = f.Package
} else if p.PackageName != f.Package {
- error(noPos, "inconsistent package names: %s, %s", p.PackageName, f.Package)
+ error(token.NoPos, "inconsistent package names: %s, %s", p.PackageName, f.Package)
}
if p.Typedef == nil {
if p.Typedef[k] == nil {
p.Typedef[k] = v
} else if !reflect.DeepEqual(p.Typedef[k], v) {
- error(noPos, "inconsistent definitions for C type %s", k)
+ error(token.NoPos, "inconsistent definitions for C type %s", k)
}
}
}
if p.Name[k] == nil {
p.Name[k] = v
} else if !reflect.DeepEqual(p.Name[k], v) {
- error(noPos, "inconsistent definitions for C.%s", k)
+ error(token.NoPos, "inconsistent definitions for C.%s", k)
}
}
}
for name, def := range p.Typedef {
fmt.Fprintf(fgo2, "type %s ", name)
- printer.Fprint(fgo2, def)
+ printer.Fprint(fgo2, fset, def)
fmt.Fprintf(fgo2, "\n")
}
fmt.Fprintf(fgo2, "type _Ctype_void [0]byte\n")
}
fmt.Fprintf(fc, "#pragma dynimport ·%s %s \"%s%s.so\"\n", n.Mangle, n.C, soprefix, sopath)
fmt.Fprintf(fgo2, "var %s ", n.Mangle)
- printer.Fprint(fgo2, &ast.StarExpr{X: n.Type.Go})
+ printer.Fprint(fgo2, fset, &ast.StarExpr{X: n.Type.Go})
fmt.Fprintf(fgo2, "\n")
}
fmt.Fprintf(fc, "\n")
Name: ast.NewIdent(n.Mangle),
Type: gtype,
}
- printer.Fprint(fgo2, d)
+ printer.Fprint(fgo2, fset, d)
fmt.Fprintf(fgo2, "\n")
if name == "CString" || name == "GoString" || name == "GoStringN" {
// Write Go output: Go input with rewrites of C.xxx to _C_xxx.
fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n")
fmt.Fprintf(fgo1, "//line %s:1\n", srcfile)
- printer.Fprint(fgo1, f.AST)
+ printer.Fprint(fgo1, fset, f.AST)
// While we process the vars and funcs, also write 6c and gcc output.
// Gcc output starts with the preamble.
// a Go wrapper function.
if fn.Recv != nil {
fmt.Fprintf(fgo2, "func %s(recv ", goname)
- printer.Fprint(fgo2, fn.Recv.List[0].Type)
+ printer.Fprint(fgo2, fset, fn.Recv.List[0].Type)
forFieldList(fntype.Params,
func(i int, atype ast.Expr) {
fmt.Fprintf(fgo2, ", p%d ", i)
- printer.Fprint(fgo2, atype)
+ printer.Fprint(fgo2, fset, atype)
})
fmt.Fprintf(fgo2, ")")
if gccResult != "void" {
if i > 0 {
fmt.Fprint(fgo2, ", ")
}
- printer.Fprint(fgo2, atype)
+ printer.Fprint(fgo2, fset, atype)
})
fmt.Fprint(fgo2, ")")
}
}
var nerrors int
-var noPos token.Position
-func error(pos token.Position, msg string, args ...interface{}) {
+func error(pos token.Pos, msg string, args ...interface{}) {
nerrors++
if pos.IsValid() {
- fmt.Fprintf(os.Stderr, "%s: ", pos)
+ fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String())
}
fmt.Fprintf(os.Stderr, msg, args...)
fmt.Fprintf(os.Stderr, "\n")
"flag"
"fmt"
"go/scanner"
+ "go/token"
"io/ioutil"
"os"
"path"
)
+var fset = token.NewFileSet()
var start = flag.String("start", "Start", "name of start production")
src = extractEBNF(src)
}
- grammar, err := ebnf.Parse(filename, src)
+ grammar, err := ebnf.Parse(fset, filename, src)
if err != nil {
scanner.PrintError(os.Stderr, err)
}
- if err = ebnf.Verify(grammar, *start); err != nil {
+ if err = ebnf.Verify(fset, grammar, *start); err != nil {
scanner.PrintError(os.Stderr, err)
}
}
"bytes"
"go/doc"
"go/parser"
+ "go/token"
"io/ioutil"
"os"
pathutil "path"
}
-func (b *treeBuilder) newDirTree(path, name string, depth int) *Directory {
+func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth int) *Directory {
if b.pathFilter != nil && !b.pathFilter(path) {
return nil
}
// though the directory doesn't contain any real package files - was bug)
if synopses[0] == "" {
// no "optimal" package synopsis yet; continue to collect synopses
- file, err := parser.ParseFile(pathutil.Join(path, d.Name), nil,
+ file, err := parser.ParseFile(fset, pathutil.Join(path, d.Name), nil,
parser.ParseComments|parser.PackageClauseOnly)
if err == nil {
hasPkgFiles = true
i := 0
for _, d := range list {
if isPkgDir(d) {
- dd := b.newDirTree(pathutil.Join(path, d.Name), d.Name, depth+1)
+ dd := b.newDirTree(fset, pathutil.Join(path, d.Name), d.Name, depth+1)
if dd != nil {
dirs[i] = dd
i++
maxDepth = 1e6 // "infinity"
}
b := treeBuilder{pathFilter, maxDepth}
- return b.newDirTree(root, d.Name, 0)
+ // the file set provided is only for local parsing, no position
+ // information escapes and thus we don't need to save the set
+ return b.newDirTree(token.NewFileSet(), root, d.Name, 0)
}
pathutil "path"
"regexp"
"runtime"
+ "sort"
"strings"
"template"
"time"
}
-// binarySearch returns an index i such that (a[i] <= s < a[i+1]) || (s is not in a).
-// The slice a must not be empty and sorted in increasing order.
-// (See "A Method of Programming", E.W. Dijkstra).
-//
-func binarySearch(a []string, s string) int {
- i, j := 0, len(a)
- // i < j for non-empty a
- for i+1 < j {
- // 0 <= i < j <= len(a) && (a[i] <= s < a[j] || (s is not in a))
- h := i + (j-i)/2 // i < h < j
- if a[h] <= s {
- i = h
- } else { // s < a[h]
- j = h
- }
- }
- // i+1 == j for non-empty a
- return i
-}
-
-
func setPathFilter(list []string) {
if len(list) == 0 {
pathFilter.set(nil)
// len(list) > 0
pathFilter.set(func(path string) bool {
// list is sorted in increasing order and for each path all its children are removed
- i := binarySearch(list, path)
- // At this point we have (list[i] <= path < list[i+1]) || (path is not in list),
- // thus path must be either longer (a child) of list[i], or shorter (a parent)
- // of list[i+1] - assuming an "infinitely extended" list. However, binarySearch
- // will return a 0 if path < list[0], so we must be careful in that case.
- return i == 0 && isParentOf(path, list[0]) ||
- isParentOf(list[i], path) ||
- i+1 < len(list) && isParentOf(path, list[i+1])
+ i := sort.Search(len(list), func(i int) bool { return list[i] > path })
+ // Now we have list[i-1] <= path < list[i].
+ // Path may be a child of list[i-1] or a parent of list[i].
+ return i > 0 && isParentOf(list[i-1], path) || i < len(list) && isParentOf(path, list[i])
})
}
// writeObjInfo writes the popup info corresponding to obj to w.
// The text is HTML-escaped and does not contain single quotes.
-func writeObjInfo(w io.Writer, obj *ast.Object) {
+func writeObjInfo(w io.Writer, fset *token.FileSet, obj *ast.Object) {
// for now, show object kind and name; eventually
// do something more interesting (show declaration,
// for instance)
// show type if we know it
if obj.Type != nil && obj.Type.Expr != nil {
fmt.Fprint(w, " ")
- writeNode(&aposescaper{w}, obj.Type.Expr, true, &defaultStyler)
+ writeNode(&aposescaper{w}, fset, obj.Type.Expr, true, &defaultStyler)
}
}
// information: The i'th array entry is a single-quoted string with
// the popup information for an identifier x with s.identId(x) == i,
// for 0 <= i < s.idcount.
-func (s *Styler) idList() []byte {
+func (s *Styler) idList(fset *token.FileSet) []byte {
var buf bytes.Buffer
fmt.Fprintln(&buf, "[")
fmt.Fprintf(&buf, "/* %4d */ ", id)
}
fmt.Fprint(&buf, "'")
- writeObjInfo(&buf, obj)
+ writeObjInfo(&buf, fset, obj)
fmt.Fprint(&buf, "',\n")
}
}
// Templates
// Write an AST-node to w; optionally html-escaped.
-func writeNode(w io.Writer, node interface{}, html bool, styler printer.Styler) {
+func writeNode(w io.Writer, fset *token.FileSet, node interface{}, html bool, styler printer.Styler) {
mode := printer.TabIndent | printer.UseSpaces
if html {
mode |= printer.GenHTML
// to ensure a good outcome in most browsers (there may still
// be tabs in comments and strings, but converting those into
// the right number of spaces is much harder)
- (&printer.Config{mode, *tabwidth, styler}).Fprint(&tconv{output: w}, node)
+ (&printer.Config{mode, *tabwidth, styler}).Fprint(&tconv{output: w}, fset, node)
}
// Write anything to w; optionally html-escaped.
-func writeAny(w io.Writer, html bool, x interface{}) {
+func writeAny(w io.Writer, fset *token.FileSet, html bool, x interface{}) {
switch v := x.(type) {
case []byte:
writeText(w, v, html)
case string:
writeText(w, []byte(v), html)
case ast.Decl, ast.Expr, ast.Stmt, *ast.File:
- writeNode(w, x, html, &defaultStyler)
+ writeNode(w, fset, x, html, &defaultStyler)
default:
if html {
var buf bytes.Buffer
}
+func fileset(x []interface{}) *token.FileSet {
+ if len(x) > 1 {
+ if fset, ok := x[1].(*token.FileSet); ok {
+ return fset
+ }
+ }
+ return nil
+}
+
+
// Template formatter for "html" format.
func htmlFmt(w io.Writer, format string, x ...interface{}) {
- writeAny(w, true, x[0])
+ writeAny(w, fileset(x), true, x[0])
}
// Template formatter for "html-esc" format.
func htmlEscFmt(w io.Writer, format string, x ...interface{}) {
var buf bytes.Buffer
- writeAny(&buf, false, x[0])
+ writeAny(&buf, fileset(x), false, x[0])
template.HTMLEscape(w, buf.Bytes())
}
// Template formatter for "html-comment" format.
func htmlCommentFmt(w io.Writer, format string, x ...interface{}) {
var buf bytes.Buffer
- writeAny(&buf, false, x[0])
+ writeAny(&buf, fileset(x), false, x[0])
// TODO(gri) Provide list of words (e.g. function parameters)
// to be emphasized by ToHTML.
doc.ToHTML(w, buf.Bytes(), nil) // does html-escaping
// Template formatter for "" (default) format.
func textFmt(w io.Writer, format string, x ...interface{}) {
- writeAny(w, false, x[0])
+ writeAny(w, fileset(x), false, x[0])
}
// determine path and position info, if any
type positioner interface {
- Pos() token.Position
+ Pos() token.Pos
}
switch t := x[0].(type) {
case string:
case positioner:
pos := t.Pos()
if pos.IsValid() {
+ pos := fileset(x).Position(pos)
path = pos.Filename
line = pos.Line
}
+ default:
+ // we should never reach here, but be resilient
+ // and assume the position is invalid (empty path,
+ // and line 0)
+ log.Printf("INTERNAL ERROR: urlFmt(%s) without a string or positioner", format)
}
// map path
func serveGoSource(w http.ResponseWriter, r *http.Request, abspath, relpath string) {
- file, err := parser.ParseFile(abspath, nil, parser.ParseComments)
+ fset := token.NewFileSet()
+ file, err := parser.ParseFile(fset, abspath, nil, parser.ParseComments)
if err != nil {
log.Printf("parser.ParseFile: %s", err)
serveError(w, r, relpath, err)
var buf bytes.Buffer
styler := newStyler(r.FormValue("h"))
- writeNode(&buf, file, true, styler)
+ writeNode(&buf, fset, file, true, styler)
type SourceInfo struct {
IdList []byte
Source []byte
}
- info := &SourceInfo{styler.idList(), buf.Bytes()}
+ info := &SourceInfo{styler.idList(fset), buf.Bytes()}
contents := applyTemplate(sourceHTML, "sourceHTML", info)
servePage(w, "Source file "+relpath, "", "", contents)
type PageInfo struct {
Dirname string // directory containing the package
PList []string // list of package names found
+ FSet *token.FileSet // corresponding file set
PAst *ast.File // nil if no single AST with package exports
PDoc *doc.PackageDoc // nil if no single package documentation
Dirs *DirList // nil if no directory information
}
// get package ASTs
- pkgs, err := parser.ParseDir(abspath, filter, parser.ParseComments)
+ fset := token.NewFileSet()
+ pkgs, err := parser.ParseDir(fset, abspath, filter, parser.ParseComments)
if err != nil && pkgs == nil {
// only report directory read errors, ignore parse errors
// (may be able to extract partial package information)
timestamp = time.Seconds()
}
- return PageInfo{abspath, plist, past, pdoc, dir.listing(true), timestamp, h.isPkg, nil}
+ return PageInfo{abspath, plist, fset, past, pdoc, dir.listing(true), timestamp, h.isPkg, nil}
}
// interface for walking file trees, and the ast.Visitor interface for
// walking Go ASTs.
type Indexer struct {
+ fset *token.FileSet // file set for all indexed files
words map[string]*IndexResult // RunLists of Spots
snippets vector.Vector // vector of *Snippets, indexed by snippet indices
file *File // current file
if kind == Use || x.decl == nil {
// not a declaration or no snippet required
- info := makeSpotInfo(kind, id.Pos().Line, false)
+ info := makeSpotInfo(kind, x.fset.Position(id.Pos()).Line, false)
lists.Others.Push(Spot{x.file, info})
} else {
// a declaration with snippet
- index := x.addSnippet(NewSnippet(x.decl, id))
+ index := x.addSnippet(NewSnippet(x.fset, x.decl, id))
info := makeSpotInfo(kind, index, true)
lists.Decls.Push(Spot{x.file, info})
}
}
-func pkgName(filename string) string {
- file, err := parser.ParseFile(filename, nil, parser.PackageClauseOnly)
+func pkgName(fset *token.FileSet, filename string) string {
+ file, err := parser.ParseFile(fset, filename, nil, parser.PackageClauseOnly)
if err != nil || file == nil {
return ""
}
return
}
- if excludeMainPackages && pkgName(path) == "main" {
+ if excludeMainPackages && pkgName(x.fset, path) == "main" {
return
}
- file, err := parser.ParseFile(path, nil, parser.ParseComments)
+ file, err := parser.ParseFile(x.fset, path, nil, parser.ParseComments)
if err != nil {
return // ignore files with (parse) errors
}
var x Indexer
// initialize Indexer
+ x.fset = token.NewFileSet()
x.words = make(map[string]*IndexResult)
// index all files in the directories given by dirnames
}
}
+ // the file set is not needed after indexing - help GC and clear it
+ x.fset = nil
+
// for each word, reduce the RunLists into a LookupResult;
// also collect the word with its canonical spelling in a
// word list for later computation of alternative spellings
func isIdentifier(s string) bool {
var S scanner.Scanner
- S.Init("", []byte(s), nil, 0)
+ S.Init(token.NewFileSet(), "", []byte(s), nil, 0)
if _, tok, _ := S.Scan(); tok == token.IDENT {
_, tok, _ := S.Scan()
return tok == token.EOF
if i > 0 {
fmt.Println()
}
- writeAny(os.Stdout, *html, d)
+ writeAny(os.Stdout, info.FSet, *html, d)
fmt.Println()
}
return
import (
"bytes"
"go/ast"
+ "go/token"
"go/printer"
"fmt"
)
}
-func newSnippet(decl ast.Decl, id *ast.Ident) *Snippet {
+func newSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) *Snippet {
var buf bytes.Buffer
- writeNode(&buf, decl, true, &snippetStyler{highlight: id})
- return &Snippet{id.Pos().Line, buf.String()}
+ writeNode(&buf, fset, decl, true, &snippetStyler{highlight: id})
+ return &Snippet{fset.Position(id.Pos()).Line, buf.String()}
}
}
-func genSnippet(d *ast.GenDecl, id *ast.Ident) *Snippet {
+func genSnippet(fset *token.FileSet, d *ast.GenDecl, id *ast.Ident) *Snippet {
s := findSpec(d.Specs, id)
if s == nil {
return nil // declaration doesn't contain id - exit gracefully
// only use the spec containing the id for the snippet
dd := &ast.GenDecl{d.Doc, d.Pos(), d.Tok, d.Lparen, []ast.Spec{s}, d.Rparen}
- return newSnippet(dd, id)
+ return newSnippet(fset, dd, id)
}
-func funcSnippet(d *ast.FuncDecl, id *ast.Ident) *Snippet {
+func funcSnippet(fset *token.FileSet, d *ast.FuncDecl, id *ast.Ident) *Snippet {
if d.Name != id {
return nil // declaration doesn't contain id - exit gracefully
}
// only use the function signature for the snippet
dd := &ast.FuncDecl{d.Doc, d.Recv, d.Name, d.Type, nil}
- return newSnippet(dd, id)
+ return newSnippet(fset, dd, id)
}
// identifier id. Parts of the declaration not containing the identifier
// may be removed for a more compact snippet.
//
-func NewSnippet(decl ast.Decl, id *ast.Ident) (s *Snippet) {
+func NewSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) (s *Snippet) {
switch d := decl.(type) {
case *ast.GenDecl:
- s = genSnippet(d, id)
+ s = genSnippet(fset, d, id)
case *ast.FuncDecl:
- s = funcSnippet(d, id)
+ s = funcSnippet(fset, d, id)
}
// handle failure gracefully
if s == nil {
s = &Snippet{
- id.Pos().Line,
+ fset.Position(id.Pos()).Line,
fmt.Sprintf(`could not generate a snippet for <span class="highlight">%s</span>`, id.Name),
}
}
type ebnfParser struct {
- out io.Writer // parser output
- src []byte // parser source
+ out io.Writer // parser output
+ src []byte // parser source
+ file *token.File // for position information
scanner scanner.Scanner
- prev int // offset of previous token
- pos token.Position // token position
- tok token.Token // one token look-ahead
- lit []byte // token literal
+ prev int // offset of previous token
+ pos token.Pos // token position
+ tok token.Token // one token look-ahead
+ lit []byte // token literal
}
func (p *ebnfParser) flush() {
- p.out.Write(p.src[p.prev:p.pos.Offset])
- p.prev = p.pos.Offset
+ offs := p.file.Offset(p.pos)
+ p.out.Write(p.src[p.prev:offs])
+ p.prev = offs
}
}
-func (p *ebnfParser) errorExpected(pos token.Position, msg string) {
+func (p *ebnfParser) errorExpected(pos token.Pos, msg string) {
msg = "expected " + msg
- if pos.Offset == p.pos.Offset {
+ if pos == p.pos {
// the error happened at the current position;
// make the error message more specific
msg += ", found '" + p.tok.String() + "'"
msg += " " + string(p.lit)
}
}
- p.Error(pos, msg)
+ p.Error(p.file.Position(pos), msg)
}
-func (p *ebnfParser) expect(tok token.Token) token.Position {
+func (p *ebnfParser) expect(tok token.Token) token.Pos {
pos := p.pos
if p.tok != tok {
p.errorExpected(pos, "'"+tok.String()+"'")
}
-func (p *ebnfParser) parse(out io.Writer, src []byte) {
+func (p *ebnfParser) parse(fset *token.FileSet, out io.Writer, src []byte) {
// initialize ebnfParser
p.out = out
p.src = src
- p.scanner.Init("", src, p, 0)
+ p.file = p.scanner.Init(fset, "", src, p, 0)
p.next() // initializes pos, tok, lit
// process source
func linkify(out io.Writer, src []byte) {
+ fset := token.NewFileSet()
for len(src) > 0 {
n := len(src)
out.Write(src[0:i])
// parse and write EBNF
var p ebnfParser
- p.parse(out, src[i:j])
+ p.parse(fset, out, src[i:j])
// advance
src = src[j:n]
"go/parser"
"go/printer"
"go/scanner"
+ "go/token"
"io/ioutil"
"os"
pathutil "path"
var (
+ fset = token.NewFileSet()
exitCode = 0
rewrite func(*ast.File) *ast.File
parserMode uint
return err
}
- file, err := parser.ParseFile(f.Name(), src, parserMode)
+ file, err := parser.ParseFile(fset, f.Name(), src, parserMode)
if err != nil {
return err
}
var res bytes.Buffer
- _, err = (&printer.Config{printerMode, *tabWidth, nil}).Fprint(&res, file)
+ _, err = (&printer.Config{printerMode, *tabWidth, nil}).Fprint(&res, fset, file)
if err != nil {
return err
}
// but there are problems with preserving formatting and also
// with what a wildcard for a statement looks like.
func parseExpr(s string, what string) ast.Expr {
- x, err := parser.ParseExpr("input", s)
+ x, err := parser.ParseExpr(fset, "input", s)
if err != nil {
fmt.Fprintf(os.Stderr, "parsing %s %s: %s\n", what, s, err)
os.Exit(2)
}
-var positionType = reflect.Typeof(token.Position{})
+var positionType = reflect.Typeof(token.NoPos)
var identType = reflect.Typeof((*ast.Ident)(nil))
"exec"
"flag"
"fmt"
+ "go/token"
"io"
"io/ioutil"
"os"
}
var (
+ fset = token.NewFileSet()
argv0 = os.Args[0]
errors = false
parents = make(map[string]string)
continue
}
filename := path.Join(dir, d.Name)
- pf, err := parser.ParseFile(filename, nil, parser.ImportsOnly)
+ pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly)
if err != nil {
return nil, nil, "", err
}
// An Expression node represents a production expression.
Expression interface {
// Pos is the position of the first character of the syntactic construct
- Pos() token.Position
+ Pos() token.Pos
}
// An Alternative node represents a non-empty list of alternative expressions.
// A Name node represents a production name.
Name struct {
- token.Position
- String string
+ StringPos token.Pos
+ String string
}
// A Token node represents a literal.
Token struct {
- token.Position
- String string
+ StringPos token.Pos
+ String string
}
// A List node represents a range of characters.
// A Group node represents a grouped expression.
Group struct {
- token.Position
- Body Expression // (body)
+ Lparen token.Pos
+ Body Expression // (body)
}
// An Option node represents an optional expression.
Option struct {
- token.Position
- Body Expression // [body]
+ Lbrack token.Pos
+ Body Expression // [body]
}
// A Repetition node represents a repeated expression.
Repetition struct {
- token.Position
- Body Expression // {body}
+ Lbrace token.Pos
+ Body Expression // {body}
}
// A Production node represents an EBNF production.
)
-func (x Alternative) Pos() token.Position {
- return x[0].Pos() // the parser always generates non-empty Alternative
-}
-
-
-func (x Sequence) Pos() token.Position {
- return x[0].Pos() // the parser always generates non-empty Sequences
-}
-
-
-func (x Range) Pos() token.Position { return x.Begin.Pos() }
-
-
-func (p *Production) Pos() token.Position { return p.Name.Pos() }
+func (x Alternative) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Alternative
+func (x Sequence) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Sequences
+func (x *Name) Pos() token.Pos { return x.StringPos }
+func (x *Token) Pos() token.Pos { return x.StringPos }
+func (x *Range) Pos() token.Pos { return x.Begin.Pos() }
+func (x *Group) Pos() token.Pos { return x.Lparen }
+func (x *Option) Pos() token.Pos { return x.Lbrack }
+func (x *Repetition) Pos() token.Pos { return x.Lbrace }
+func (x *Production) Pos() token.Pos { return x.Name.Pos() }
// ----------------------------------------------------------------------------
type verifier struct {
+ fset *token.FileSet
scanner.ErrorVector
worklist []*Production
reached Grammar // set of productions reached from (and including) the root production
}
+func (v *verifier) error(pos token.Pos, msg string) {
+ v.Error(v.fset.Position(pos), msg)
+}
+
+
func (v *verifier) push(prod *Production) {
name := prod.Name.String
if _, found := v.reached[name]; !found {
func (v *verifier) verifyChar(x *Token) int {
s := x.String
if utf8.RuneCountInString(s) != 1 {
- v.Error(x.Pos(), "single char expected, found "+s)
+ v.error(x.Pos(), "single char expected, found "+s)
return 0
}
ch, _ := utf8.DecodeRuneInString(s)
if prod, found := v.grammar[x.String]; found {
v.push(prod)
} else {
- v.Error(x.Pos(), "missing production "+x.String)
+ v.error(x.Pos(), "missing production "+x.String)
}
// within a lexical production references
// to non-lexical productions are invalid
if lexical && !isLexical(x.String) {
- v.Error(x.Pos(), "reference to non-lexical production "+x.String)
+ v.error(x.Pos(), "reference to non-lexical production "+x.String)
}
case *Token:
// nothing to do for now
i := v.verifyChar(x.Begin)
j := v.verifyChar(x.End)
if i >= j {
- v.Error(x.Pos(), "decreasing character range")
+ v.error(x.Pos(), "decreasing character range")
}
case *Group:
v.verifyExpr(x.Body, lexical)
}
-func (v *verifier) verify(grammar Grammar, start string) {
+func (v *verifier) verify(fset *token.FileSet, grammar Grammar, start string) {
// find root production
root, found := grammar[start]
if !found {
- var noPos token.Position
- v.Error(noPos, "no start production "+start)
+ // token.NoPos doesn't require a file set;
+ // ok to set v.fset only afterwards
+ v.error(token.NoPos, "no start production "+start)
return
}
// initialize verifier
+ v.fset = fset
v.ErrorVector.Reset()
v.worklist = v.worklist[0:0]
v.reached = make(Grammar)
if len(v.reached) < len(v.grammar) {
for name, prod := range v.grammar {
if _, found := v.reached[name]; !found {
- v.Error(prod.Pos(), name+" is unreachable")
+ v.error(prod.Pos(), name+" is unreachable")
}
}
}
// - all productions defined are used when beginning at start
// - lexical productions refer only to other lexical productions
//
-func Verify(grammar Grammar, start string) os.Error {
+// Position information is interpreted relative to the file set fset.
+//
+func Verify(fset *token.FileSet, grammar Grammar, start string) os.Error {
var v verifier
- v.verify(grammar, start)
+ v.verify(fset, grammar, start)
return v.GetError(scanner.Sorted)
}
package ebnf
import (
+ "go/token"
"io/ioutil"
"testing"
)
+var fset = token.NewFileSet()
+
+
var grammars = []string{
`Program = .
`,
func check(t *testing.T, filename string, src []byte) {
- grammar, err := Parse(filename, src)
+ grammar, err := Parse(fset, filename, src)
if err != nil {
t.Errorf("Parse(%s) failed: %v", src, err)
}
- if err = Verify(grammar, "Program"); err != nil {
+ if err = Verify(fset, grammar, "Program"); err != nil {
t.Errorf("Verify(%s) failed: %v", src, err)
}
}
type parser struct {
+ fset *token.FileSet
scanner.ErrorVector
scanner scanner.Scanner
- pos token.Position // token position
- tok token.Token // one token look-ahead
- lit []byte // token literal
+ pos token.Pos // token position
+ tok token.Token // one token look-ahead
+ lit []byte // token literal
}
}
-func (p *parser) errorExpected(pos token.Position, msg string) {
+func (p *parser) error(pos token.Pos, msg string) {
+ p.Error(p.fset.Position(pos), msg)
+}
+
+
+func (p *parser) errorExpected(pos token.Pos, msg string) {
msg = "expected " + msg
- if pos.Offset == p.pos.Offset {
+ if pos == p.pos {
// the error happened at the current position;
// make the error message more specific
msg += ", found '" + p.tok.String() + "'"
msg += " " + string(p.lit)
}
}
- p.Error(pos, msg)
+ p.error(pos, msg)
}
-func (p *parser) expect(tok token.Token) token.Position {
+func (p *parser) expect(tok token.Token) token.Pos {
pos := p.pos
if p.tok != tok {
p.errorExpected(pos, "'"+tok.String()+"'")
}
-func (p *parser) parse(filename string, src []byte) Grammar {
+func (p *parser) parse(fset *token.FileSet, filename string, src []byte) Grammar {
// initialize parser
+ p.fset = fset
p.ErrorVector.Reset()
- p.scanner.Init(filename, src, p, 0)
+ p.scanner.Init(fset, filename, src, p, 0)
p.next() // initializes pos, tok, lit
grammar := make(Grammar)
if _, found := grammar[name]; !found {
grammar[name] = prod
} else {
- p.Error(prod.Pos(), name+" declared already")
+ p.error(prod.Pos(), name+" declared already")
}
}
// Parse parses a set of EBNF productions from source src.
// It returns a set of productions. Errors are reported
// for incorrect syntax and if a production is declared
-// more than once.
+// more than once. Position information is recorded relative
+// to the file set fset.
//
-func Parse(filename string, src []byte) (Grammar, os.Error) {
+func Parse(fset *token.FileSet, filename string, src []byte) (Grammar, os.Error) {
var p parser
- grammar := p.parse(filename, src)
+ grammar := p.parse(fset, filename, src)
return grammar, p.GetError(scanner.Sorted)
}
import (
"fmt"
"testing"
+ "go/token"
)
+var fset = token.NewFileSet()
+
+
func parse(t *testing.T, form string, fmap FormatterMap) Format {
- f, err := Parse("", []byte(form), fmap)
+ f, err := Parse(fset, "", []byte(form), fmap)
if err != nil {
t.Errorf("Parse(%s): %v", form, err)
return nil
type parser struct {
scanner.ErrorVector
scanner scanner.Scanner
- pos token.Position // token position
- tok token.Token // one token look-ahead
- lit []byte // token literal
+ file *token.File
+ pos token.Pos // token position
+ tok token.Token // one token look-ahead
+ lit []byte // token literal
packs map[string]string // PackageName -> ImportPath
rules map[string]expr // RuleName -> Expression
}
-func (p *parser) init(filename string, src []byte) {
+func (p *parser) init(fset *token.FileSet, filename string, src []byte) {
p.ErrorVector.Reset()
- p.scanner.Init(filename, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message
- p.next() // initializes pos, tok, lit
+ p.file = p.scanner.Init(fset, filename, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message
+ p.next() // initializes pos, tok, lit
p.packs = make(map[string]string)
p.rules = make(map[string]expr)
}
-func (p *parser) errorExpected(pos token.Position, msg string) {
+func (p *parser) error(pos token.Pos, msg string) {
+ p.Error(p.file.Position(pos), msg)
+}
+
+
+func (p *parser) errorExpected(pos token.Pos, msg string) {
msg = "expected " + msg
- if pos.Offset == p.pos.Offset {
+ if pos == p.pos {
// the error happened at the current position;
// make the error message more specific
msg += ", found '" + p.tok.String() + "'"
msg += " " + string(p.lit)
}
}
- p.Error(pos, msg)
+ p.error(pos, msg)
}
-func (p *parser) expect(tok token.Token) token.Position {
+func (p *parser) expect(tok token.Token) token.Pos {
pos := p.pos
if p.tok != tok {
p.errorExpected(pos, "'"+tok.String()+"'")
if importPath, found := p.packs[name]; found {
name = importPath
} else {
- p.Error(pos, "package not declared: "+name)
+ p.error(pos, "package not declared: "+name)
}
p.next()
name, isIdent = name+"."+p.parseIdentifier(), false
// add package declaration
if !isIdent {
- p.Error(pos, "illegal package name: "+name)
+ p.error(pos, "illegal package name: "+name)
} else if _, found := p.packs[name]; !found {
p.packs[name] = importPath
} else {
- p.Error(pos, "package already declared: "+name)
+ p.error(pos, "package already declared: "+name)
}
case token.ASSIGN:
if _, found := p.rules[name]; !found {
p.rules[name] = x
} else {
- p.Error(pos, "format rule already declared: "+name)
+ p.error(pos, "format rule already declared: "+name)
}
default:
// there are no errors, the result is a Format and the error is nil.
// Otherwise the format is nil and a non-empty ErrorList is returned.
//
-func Parse(filename string, src []byte, fmap FormatterMap) (Format, os.Error) {
+func Parse(fset *token.FileSet, filename string, src []byte, fmap FormatterMap) (Format, os.Error) {
// parse source
var p parser
- p.init(filename, src)
+ p.init(fset, filename, src)
p.parseFormat()
// add custom formatters, if any
var nt *NamedType
if t.Name() != "" {
name := t.PkgPath() + "·" + t.Name()
- nt = &NamedType{token.Position{}, name, nil, true, make(map[string]Method)}
+ nt = &NamedType{token.NoPos, name, nil, true, make(map[string]Method)}
evalTypes[t] = nt
}
)
-type positioned interface {
- Pos() token.Position
-}
-
-
// A compiler captures information used throughout an entire
// compilation. Currently it includes only the error handler.
//
// TODO(austin) This might actually represent package level, in which
// case it should be package compiler.
type compiler struct {
+ fset *token.FileSet
errors scanner.ErrorHandler
numErrors int
silentErrors int
}
-func (a *compiler) diagAt(pos positioned, format string, args ...interface{}) {
- a.errors.Error(pos.Pos(), fmt.Sprintf(format, args...))
+func (a *compiler) diagAt(pos token.Pos, format string, args ...interface{}) {
+ a.errors.Error(a.fset.Position(pos), fmt.Sprintf(format, args...))
a.numErrors++
}
continuePC *uint
// The position where this label was resolved. If it has not
// been resolved yet, an invalid position.
- resolved token.Position
+ resolved token.Pos
// The position where this label was first jumped to.
- used token.Position
+ used token.Pos
}
// A funcCompiler captures information used throughout the compilation
"big"
"flag"
"fmt"
+ "go/token"
"log"
"os"
"reflect"
"testing"
)
+// All tests are done using the same file set.
+var fset = token.NewFileSet()
+
// Print each statement or expression before parsing it
var noisy = false
println("code:", src)
}
- code, err := w.Compile(src)
+ code, err := w.Compile(fset, src)
if err != nil {
if j.cerr == "" {
t.Errorf("%s: Compile %s: %v", name, src, err)
// compiled from it.
type exprInfo struct {
*compiler
- pos token.Position
+ pos token.Pos
}
func (a *exprInfo) newExpr(t Type, desc string) *expr {
}
func (a *exprInfo) diag(format string, args ...interface{}) {
- a.diagAt(&a.pos, format, args...)
+ a.diagAt(a.pos, format, args...)
}
func (a *exprInfo) diagOpType(op token.Token, vt Type) {
// multi-valued type.
type assignCompiler struct {
*compiler
- pos token.Position
+ pos token.Pos
// The RHS expressions. This may include nil's for
// expressions that failed to compile.
rs []*expr
// assignCompiler with rmt set, but if type checking fails, slots in
// the MultiType may be nil. If rs contains nil's, type checking will
// fail and these expressions given a nil type.
-func (a *compiler) checkAssign(pos token.Position, rs []*expr, errOp, errPosName string) (*assignCompiler, bool) {
+func (a *compiler) checkAssign(pos token.Pos, rs []*expr, errOp, errPosName string) (*assignCompiler, bool) {
c := &assignCompiler{
compiler: a,
pos: pos,
pos = a.rs[lcount-1].pos
}
}
- a.diagAt(&pos, "%s %ss for %s\n\t%s\n\t%s", msg, a.errPosName, a.errOp, lt, rmt)
+ a.diagAt(pos, "%s %ss for %s\n\t%s\n\t%s", msg, a.errPosName, a.errOp, lt, rmt)
return nil
}
// compileAssign compiles an assignment operation without the full
// generality of an assignCompiler. See assignCompiler for a
// description of the arguments.
-func (a *compiler) compileAssign(pos token.Position, b *block, lt Type, rs []*expr, errOp, errPosName string) func(Value, *Thread) {
+func (a *compiler) compileAssign(pos token.Pos, b *block, lt Type, rs []*expr, errOp, errPosName string) func(Value, *Thread) {
ac, ok := a.checkAssign(pos, rs, errOp, errPosName)
if !ok {
return nil
return nil
}
if a.constant {
- a.diagAt(x, "function literal used in constant expression")
+ a.diagAt(x.Pos(), "function literal used in constant expression")
return nil
}
return ei.compileFuncLit(decl, fn)
return nil
}
if a.constant {
- a.diagAt(x, "function call in constant context")
+ a.diagAt(x.Pos(), "function call in constant context")
return nil
}
if l.valType != nil {
- a.diagAt(x, "type conversions not implemented")
+ a.diagAt(x.Pos(), "type conversions not implemented")
return nil
} else if ft, ok := l.t.(*FuncType); ok && ft.builtin != "" {
return ei.compileBuiltinCallExpr(a.block, ft, args)
typeexpr:
if !callCtx {
- a.diagAt(x, "type used as expression")
+ a.diagAt(x.Pos(), "type used as expression")
return nil
}
return ei.exprFromType(a.compileType(a.block, x))
notimpl:
- a.diagAt(x, "%T expression node not implemented", x)
+ a.diagAt(x.Pos(), "%T expression node not implemented", x)
return nil
}
}
if !lenExpr.t.isInteger() {
- a.diagAt(expr, "array size must be an integer")
+ a.diagAt(expr.Pos(), "array size must be an integer")
return 0, false
}
"flag"
"go/parser"
"go/scanner"
+ "go/token"
"io/ioutil"
"os"
)
+var fset = token.NewFileSet()
var filename = flag.String("f", "", "file to run")
func main() {
println(err.String())
os.Exit(1)
}
- file, err := parser.ParseFile(*filename, data, 0)
+ file, err := parser.ParseFile(fset, *filename, data, 0)
if err != nil {
println(err.String())
os.Exit(1)
}
- code, err := w.CompileDeclList(file.Decls)
+ code, err := w.CompileDeclList(fset, file.Decls)
if err != nil {
if list, ok := err.(scanner.ErrorList); ok {
for _, e := range list {
println(err.String())
os.Exit(1)
}
- code, err = w.Compile("init()")
+ code, err = w.Compile(fset, "init()")
if code != nil {
_, err := code.Run()
if err != nil {
os.Exit(1)
}
}
- code, err = w.Compile("main()")
+ code, err = w.Compile(fset, "main()")
if err != nil {
println(err.String())
os.Exit(1)
if err != nil {
break
}
- code, err := w.Compile(line)
+ code, err := w.Compile(fset, line)
if err != nil {
println(err.String())
continue
// A definition can be a *Variable, *Constant, or Type.
type Def interface {
- Pos() token.Position
+ Pos() token.Pos
}
type Variable struct {
- token.Position
+ VarPos token.Pos
// Index of this variable in the Frame structure
Index int
// Static type of this variable
Init Value
}
+func (v *Variable) Pos() token.Pos {
+ return v.VarPos
+}
+
type Constant struct {
- token.Position
- Type Type
- Value Value
+ ConstPos token.Pos
+ Type Type
+ Value Value
+}
+
+func (c *Constant) Pos() token.Pos {
+ return c.ConstPos
}
// A block represents a definition block in which a name may not be
return sub.scope
}
-func (b *block) DefineVar(name string, pos token.Position, t Type) (*Variable, Def) {
+func (b *block) DefineVar(name string, pos token.Pos, t Type) (*Variable, Def) {
if prev, ok := b.defs[name]; ok {
return nil, prev
}
v := b.defineSlot(t, false)
- v.Position = pos
+ v.VarPos = pos
b.defs[name] = v
return v, nil
}
b.scope.maxVars = index + 1
}
}
- v := &Variable{token.Position{}, index, t, nil}
+ v := &Variable{token.NoPos, index, t, nil}
return v
}
-func (b *block) DefineConst(name string, pos token.Position, t Type, v Value) (*Constant, Def) {
+func (b *block) DefineConst(name string, pos token.Pos, t Type, v Value) (*Constant, Def) {
if prev, ok := b.defs[name]; ok {
return nil, prev
}
return c, nil
}
-func (b *block) DefineType(name string, pos token.Position, t Type) Type {
+func (b *block) DefineType(name string, pos token.Pos, t Type) Type {
if _, ok := b.defs[name]; ok {
return nil
}
type stmtCompiler struct {
*blockCompiler
- pos token.Position
+ pos token.Pos
// This statement's label, or nil if it is not labeled.
stmtLabel *label
}
func (a *stmtCompiler) diag(format string, args ...interface{}) {
- a.diagAt(&a.pos, format, args...)
+ a.diagAt(a.pos, format, args...)
}
/*
ents map[uint]*flowEnt
// gotos is a map from goto positions to information on the
// block at the point of the goto.
- gotos map[*token.Position]*flowBlock
+ gotos map[token.Pos]*flowBlock
// labels is a map from label name to information on the block
// at the point of the label. labels are tracked by name,
// since mutliple labels at the same PC can have different
}
func newFlowBuf(cb *codeBuf) *flowBuf {
- return &flowBuf{cb, make(map[uint]*flowEnt), make(map[*token.Position]*flowBlock), make(map[string]*flowBlock)}
+ return &flowBuf{cb, make(map[uint]*flowEnt), make(map[token.Pos]*flowBlock), make(map[string]*flowBlock)}
}
// put creates a flow control point for the next PC in the code buffer.
// putGoto captures the block at a goto statement. This should be
// called in addition to putting a flow control point.
-func (f *flowBuf) putGoto(pos token.Position, target string, b *block) {
- f.gotos[&pos] = newFlowBlock(target, b)
+func (f *flowBuf) putGoto(pos token.Pos, target string, b *block) {
+ f.gotos[pos] = newFlowBlock(target, b)
}
// putLabel captures the block at a label.
func (a *stmtCompiler) defineVar(ident *ast.Ident, t Type) *Variable {
v, prev := a.block.DefineVar(ident.Name, ident.Pos(), t)
if prev != nil {
- // TODO(austin) It's silly that we have to capture
- // Pos() in a variable.
- pos := prev.Pos()
- if pos.IsValid() {
- a.diagAt(ident, "variable %s redeclared in this block\n\tprevious declaration at %s", ident.Name, &pos)
+ if prev.Pos().IsValid() {
+ a.diagAt(ident.Pos(), "variable %s redeclared in this block\n\tprevious declaration at %s", ident.Name, a.fset.Position(prev.Pos()))
} else {
- a.diagAt(ident, "variable %s redeclared in this block", ident.Name)
+ a.diagAt(ident.Pos(), "variable %s redeclared in this block", ident.Name)
}
return nil
}
if prev != nil {
pos := prev.Pos()
if pos.IsValid() {
- a.diagAt(d.Name, "identifier %s redeclared in this block\n\tprevious declaration at %s", d.Name.Name, &pos)
+ a.diagAt(d.Name.Pos(), "identifier %s redeclared in this block\n\tprevious declaration at %s", d.Name.Name, a.fset.Position(pos))
} else {
- a.diagAt(d.Name, "identifier %s redeclared in this block", d.Name.Name)
+ a.diagAt(d.Name.Pos(), "identifier %s redeclared in this block", d.Name.Name)
}
}
fn := a.compileFunc(a.block, decl, d.Body)
l, ok := a.labels[s.Label.Name]
if ok {
if l.resolved.IsValid() {
- a.diag("label %s redeclared in this block\n\tprevious declaration at %s", s.Label.Name, &l.resolved)
+ a.diag("label %s redeclared in this block\n\tprevious declaration at %s", s.Label.Name, a.fset.Position(l.resolved))
}
} else {
pc := badPC
// Check that it's an identifier
ident, ok = le.(*ast.Ident)
if !ok {
- a.diagAt(le, "left side of := must be a name")
+ a.diagAt(le.Pos(), "left side of := must be a name")
// Suppress new defitions errors
nDefs++
continue
for _, c := range s.Body.List {
clause, ok := c.(*ast.CaseClause)
if !ok {
- a.diagAt(clause, "switch statement must contain case clauses")
+ a.diagAt(clause.Pos(), "switch statement must contain case clauses")
continue
}
if clause.Values == nil {
if hasDefault {
- a.diagAt(clause, "switch statement contains more than one default case")
+ a.diagAt(clause.Pos(), "switch statement contains more than one default case")
}
hasDefault = true
} else {
case e == nil:
// Error reported by compileExpr
case cond == nil && !e.t.isBoolean():
- a.diagAt(v, "'case' condition must be boolean")
+ a.diagAt(v.Pos(), "'case' condition must be boolean")
case cond == nil:
cases[i] = e.asBool()
case cond != nil:
// empty blocks to be empty
// statements.
if _, ok := s2.(*ast.EmptyStmt); !ok {
- a.diagAt(s, "fallthrough statement must be final statement in case")
+ a.diagAt(s.Pos(), "fallthrough statement must be final statement in case")
break
}
}
// this if there were no errors compiling the body.
if len(decl.Type.Out) > 0 && fc.flow.reachesEnd(0) {
// XXX(Spec) Not specified.
- a.diagAt(&body.Rbrace, "function ends without a return statement")
+ a.diagAt(body.Rbrace, "function ends without a return statement")
return nil
}
nerr := a.numError()
for _, l := range a.labels {
if !l.resolved.IsValid() {
- a.diagAt(&l.used, "label %s not defined", l.name)
+ a.diagAt(l.used, "label %s not defined", l.name)
}
}
if nerr != a.numError() {
// String returns the string representation of this type.
String() string
// The position where this type was defined, if any.
- Pos() token.Position
+ Pos() token.Pos
}
type BoundedType interface {
maxVal() *big.Rat
}
-var universePos = token.Position{"<universe>", 0, 0, 0}
+var universePos = token.NoPos
/*
* Type array maps. These are used to memoize composite types.
func (commonType) isIdeal() bool { return false }
-func (commonType) Pos() token.Position { return token.Position{} }
+func (commonType) Pos() token.Pos { return token.NoPos }
/*
* Bool
}
type NamedType struct {
- token.Position
- Name string
+ NamePos token.Pos
+ Name string
// Underlying type. If incomplete is true, this will be nil.
// If incomplete is false and this is still nil, then this is
// a placeholder type representing an error.
// TODO(austin) This is temporarily needed by the debugger's remote
// type parser. This should only be possible with block.DefineType.
func NewNamedType(name string) *NamedType {
- return &NamedType{token.Position{}, name, nil, true, make(map[string]Method)}
+ return &NamedType{token.NoPos, name, nil, true, make(map[string]Method)}
+}
+
+func (t *NamedType) Pos() token.Pos {
+ return t.NamePos
}
func (t *NamedType) Complete(def Type) {
func (a *typeCompiler) compileIdent(x *ast.Ident, allowRec bool) Type {
_, _, def := a.block.Lookup(x.Name)
if def == nil {
- a.diagAt(x, "%s: undefined", x.Name)
+ a.diagAt(x.Pos(), "%s: undefined", x.Name)
return nil
}
switch def := def.(type) {
case *Constant:
- a.diagAt(x, "constant %v used as type", x.Name)
+ a.diagAt(x.Pos(), "constant %v used as type", x.Name)
return nil
case *Variable:
- a.diagAt(x, "variable %v used as type", x.Name)
+ a.diagAt(x.Pos(), "variable %v used as type", x.Name)
return nil
case *NamedType:
if !allowRec && def.incomplete {
- a.diagAt(x, "illegal recursive type")
+ a.diagAt(x.Pos(), "illegal recursive type")
return nil
}
if !def.incomplete && def.Def == nil {
}
if _, ok := x.Len.(*ast.Ellipsis); ok {
- a.diagAt(x.Len, "... array initailizers not implemented")
+ a.diagAt(x.Len.Pos(), "... array initailizers not implemented")
return nil
}
l, ok := a.compileArrayLen(a.block, x.Len)
return nil
}
if l < 0 {
- a.diagAt(x.Len, "array length must be non-negative")
+ a.diagAt(x.Len.Pos(), "array length must be non-negative")
return nil
}
if elem == nil {
return NewArrayType(l, elem)
}
-func (a *typeCompiler) compileFields(fields *ast.FieldList, allowRec bool) ([]Type, []*ast.Ident, []token.Position, bool) {
+func (a *typeCompiler) compileFields(fields *ast.FieldList, allowRec bool) ([]Type, []*ast.Ident, []token.Pos, bool) {
n := fields.NumFields()
ts := make([]Type, n)
ns := make([]*ast.Ident, n)
- ps := make([]token.Position, n)
+ ps := make([]token.Pos, n)
bad := false
if fields != nil {
// uniqueness of field names inherited from anonymous fields
// at use time.
fields := make([]StructField, len(ts))
- nameSet := make(map[string]token.Position, len(ts))
+ nameSet := make(map[string]token.Pos, len(ts))
for i := range fields {
// Compute field name and check anonymous fields
var name string
// *T, and T itself, may not be a pointer or
// interface type.
if nt == nil {
- a.diagAt(&poss[i], "embedded type must T or *T, where T is a named type")
+ a.diagAt(poss[i], "embedded type must T or *T, where T is a named type")
bad = true
continue
}
lateCheck := a.lateCheck
a.lateCheck = func() bool {
if _, ok := nt.lit().(*PtrType); ok {
- a.diagAt(&poss[i], "embedded type %v is a pointer type", nt)
+ a.diagAt(poss[i], "embedded type %v is a pointer type", nt)
return false
}
return lateCheck()
// Check name uniqueness
if prev, ok := nameSet[name]; ok {
- a.diagAt(&poss[i], "field %s redeclared\n\tprevious declaration at %s", name, &prev)
+ a.diagAt(poss[i], "field %s redeclared\n\tprevious declaration at %s", name, a.fset.Position(prev))
bad = true
continue
}
ts, names, poss, bad := a.compileFields(x.Methods, allowRec)
methods := make([]IMethod, len(ts))
- nameSet := make(map[string]token.Position, len(ts))
+ nameSet := make(map[string]token.Pos, len(ts))
embeds := make([]*InterfaceType, len(ts))
var nm, ne int
methods[nm].Type = ts[i].(*FuncType)
nm++
if prev, ok := nameSet[name]; ok {
- a.diagAt(&poss[i], "method %s redeclared\n\tprevious declaration at %s", name, &prev)
+ a.diagAt(poss[i], "method %s redeclared\n\tprevious declaration at %s", name, a.fset.Position(prev))
bad = true
continue
}
// Embedded interface
it, ok := ts[i].lit().(*InterfaceType)
if !ok {
- a.diagAt(&poss[i], "embedded type must be an interface")
+ a.diagAt(poss[i], "embedded type must be an interface")
bad = true
continue
}
ne++
for _, m := range it.methods {
if prev, ok := nameSet[m.Name]; ok {
- a.diagAt(&poss[i], "method %s redeclared\n\tprevious declaration at %s", m.Name, &prev)
+ a.diagAt(poss[i], "method %s redeclared\n\tprevious declaration at %s", m.Name, a.fset.Position(prev))
bad = true
continue
}
// that can be map keys except for function types.
switch key.lit().(type) {
case *StructType:
- a.diagAt(x, "map key cannot be a struct type")
+ a.diagAt(x.Pos(), "map key cannot be a struct type")
return nil
case *ArrayType:
- a.diagAt(x, "map key cannot be an array type")
+ a.diagAt(x.Pos(), "map key cannot be an array type")
return nil
case *SliceType:
- a.diagAt(x, "map key cannot be a slice type")
+ a.diagAt(x.Pos(), "map key cannot be a slice type")
return nil
}
return NewMapType(key, val)
return a.compileType(x.X, allowRec)
case *ast.Ellipsis:
- a.diagAt(x, "illegal use of ellipsis")
+ a.diagAt(x.Pos(), "illegal use of ellipsis")
return nil
}
- a.diagAt(x, "expression used as type")
+ a.diagAt(x.Pos(), "expression used as type")
return nil
notimpl:
- a.diagAt(x, "compileType: %T not implemented", x)
+ a.diagAt(x.Pos(), "compileType: %T not implemented", x)
return nil
}
code code
}
-func (w *World) CompileStmtList(stmts []ast.Stmt) (Code, os.Error) {
+func (w *World) CompileStmtList(fset *token.FileSet, stmts []ast.Stmt) (Code, os.Error) {
if len(stmts) == 1 {
if s, ok := stmts[0].(*ast.ExprStmt); ok {
- return w.CompileExpr(s.X)
+ return w.CompileExpr(fset, s.X)
}
}
errors := new(scanner.ErrorVector)
- cc := &compiler{errors, 0, 0}
+ cc := &compiler{fset, errors, 0, 0}
cb := newCodeBuf()
fc := &funcCompiler{
compiler: cc,
return &stmtCode{w, fc.get()}, nil
}
-func (w *World) CompileDeclList(decls []ast.Decl) (Code, os.Error) {
+func (w *World) CompileDeclList(fset *token.FileSet, decls []ast.Decl) (Code, os.Error) {
stmts := make([]ast.Stmt, len(decls))
for i, d := range decls {
stmts[i] = &ast.DeclStmt{d}
}
- return w.CompileStmtList(stmts)
+ return w.CompileStmtList(fset, stmts)
}
func (s *stmtCode) Type() Type { return nil }
eval func(Value, *Thread)
}
-func (w *World) CompileExpr(e ast.Expr) (Code, os.Error) {
+func (w *World) CompileExpr(fset *token.FileSet, e ast.Expr) (Code, os.Error) {
errors := new(scanner.ErrorVector)
- cc := &compiler{errors, 0, 0}
+ cc := &compiler{fset, errors, 0, 0}
ec := cc.compileExpr(w.scope.block, false, e)
if ec == nil {
return v, err
}
-func (w *World) Compile(text string) (Code, os.Error) {
- stmts, err := parser.ParseStmtList("input", text)
+func (w *World) Compile(fset *token.FileSet, text string) (Code, os.Error) {
+ stmts, err := parser.ParseStmtList(fset, "input", text)
if err == nil {
- return w.CompileStmtList(stmts)
+ return w.CompileStmtList(fset, stmts)
}
// Otherwise try as DeclList.
- decls, err1 := parser.ParseDeclList("input", text)
+ decls, err1 := parser.ParseDeclList(fset, "input", text)
if err1 == nil {
- return w.CompileDeclList(decls)
+ return w.CompileDeclList(fset, decls)
}
// Have to pick an error.
res := "identifier " + e.Name + " redeclared"
pos := e.Prev.Pos()
if pos.IsValid() {
- res += "; previous declaration at " + pos.String()
+ // TODO: fix this - currently this code is not reached by the tests
+ // need to get a file set (fset) from somewhere
+ //res += "; previous declaration at " + fset.Position(pos).String()
+ panic(0)
}
return res
}
func (w *World) DefineConst(name string, t Type, val Value) os.Error {
- _, prev := w.scope.DefineConst(name, token.Position{}, t, val)
+ _, prev := w.scope.DefineConst(name, token.NoPos, t, val)
if prev != nil {
return &RedefinitionError{name, prev}
}
}
func (w *World) DefineVar(name string, t Type, val Value) os.Error {
- v, prev := w.scope.DefineVar(name, token.Position{}, t)
+ v, prev := w.scope.DefineVar(name, token.NoPos, t)
if prev != nil {
return &RedefinitionError{name, prev}
}
"strings"
)
+var fset = token.NewFileSet()
var world *eval.World
var curProc *Process
}
// Try line as code
- code, err := world.Compile(string(line))
+ code, err := world.Compile(fset, string(line))
if err != nil {
scanner.PrintError(os.Stderr, err)
continue
func newScanner(input []byte) (*scanner.Scanner, *scanner.ErrorVector) {
sc := new(scanner.Scanner)
ev := new(scanner.ErrorVector)
- sc.Init("input", input, ev, 0)
-
+ sc.Init(fset, "input", input, ev, 0)
return sc, ev
}
slit := string(lit)
for i := range cmds {
if cmds[i].cmd == slit {
- return &cmds[i], line[pos.Offset+len(lit):]
+ return &cmds[i], line[fset.Position(pos).Offset+len(lit):]
}
}
return nil, nil
// All node types implement the Node interface.
type Node interface {
// Pos returns the (beginning) position of the node.
- Pos() token.Position
+ Pos() token.Pos
}
// A Comment node represents a single //-style or /*-style comment.
type Comment struct {
- Slash token.Position // position of "/" starting the comment
- Text []byte // comment text (excluding '\n' for //-style comments)
+ Slash token.Pos // position of "/" starting the comment
+ Text []byte // comment text (excluding '\n' for //-style comments)
}
-func (c *Comment) Pos() token.Position {
+func (c *Comment) Pos() token.Pos {
return c.Slash
}
}
-func (f *Field) Pos() token.Position {
+func (f *Field) Pos() token.Pos {
if len(f.Names) > 0 {
return f.Names[0].Pos()
}
// A FieldList represents a list of Fields, enclosed by parentheses or braces.
type FieldList struct {
- Opening token.Position // position of opening parenthesis/brace
- List []*Field // field list
- Closing token.Position // position of closing parenthesis/brace
+ Opening token.Pos // position of opening parenthesis/brace
+ List []*Field // field list
+ Closing token.Pos // position of closing parenthesis/brace
}
// created.
//
BadExpr struct {
- Begin token.Position // beginning position of bad expression
+ Begin token.Pos // beginning position of bad expression
}
// An Ident node represents an identifier.
Ident struct {
- NamePos token.Position // identifier position
- Name string // identifier name
- Obj *Object // denoted object; or nil
+ NamePos token.Pos // identifier position
+ Name string // identifier name
+ Obj *Object // denoted object; or nil
}
// An Ellipsis node stands for the "..." type in a
// parameter list or the "..." length in an array type.
//
Ellipsis struct {
- Ellipsis token.Position // position of "..."
- Elt Expr // ellipsis element type (parameter lists only)
+ Ellipsis token.Pos // position of "..."
+ Elt Expr // ellipsis element type (parameter lists only)
}
// A BasicLit node represents a literal of basic type.
BasicLit struct {
- ValuePos token.Position // literal position
- Kind token.Token // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING
- Value []byte // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o`
+ ValuePos token.Pos // literal position
+ Kind token.Token // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING
+ Value []byte // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o`
}
// A FuncLit node represents a function literal.
// A CompositeLit node represents a composite literal.
CompositeLit struct {
- Type Expr // literal type; or nil
- Lbrace token.Position // position of "{"
- Elts []Expr // list of composite elements
- Rbrace token.Position // position of "}"
+ Type Expr // literal type; or nil
+ Lbrace token.Pos // position of "{"
+ Elts []Expr // list of composite elements
+ Rbrace token.Pos // position of "}"
}
// A ParenExpr node represents a parenthesized expression.
ParenExpr struct {
- Lparen token.Position // position of "("
- X Expr // parenthesized expression
- Rparen token.Position // position of ")"
+ Lparen token.Pos // position of "("
+ X Expr // parenthesized expression
+ Rparen token.Pos // position of ")"
}
// A SelectorExpr node represents an expression followed by a selector.
// A CallExpr node represents an expression followed by an argument list.
CallExpr struct {
- Fun Expr // function expression
- Lparen token.Position // position of "("
- Args []Expr // function arguments
- Ellipsis token.Position // position of "...", if any
- Rparen token.Position // position of ")"
+ Fun Expr // function expression
+ Lparen token.Pos // position of "("
+ Args []Expr // function arguments
+ Ellipsis token.Pos // position of "...", if any
+ Rparen token.Pos // position of ")"
}
// A StarExpr node represents an expression of the form "*" Expression.
// Semantically it could be a unary "*" expression, or a pointer type.
//
StarExpr struct {
- Star token.Position // position of "*"
- X Expr // operand
+ Star token.Pos // position of "*"
+ X Expr // operand
}
// A UnaryExpr node represents a unary expression.
// Unary "*" expressions are represented via StarExpr nodes.
//
UnaryExpr struct {
- OpPos token.Position // position of Op
- Op token.Token // operator
- X Expr // operand
+ OpPos token.Pos // position of Op
+ Op token.Token // operator
+ X Expr // operand
}
// A BinaryExpr node represents a binary expression.
BinaryExpr struct {
- X Expr // left operand
- OpPos token.Position // position of Op
- Op token.Token // operator
- Y Expr // right operand
+ X Expr // left operand
+ OpPos token.Pos // position of Op
+ Op token.Token // operator
+ Y Expr // right operand
}
// A KeyValueExpr node represents (key : value) pairs
//
KeyValueExpr struct {
Key Expr
- Colon token.Position // position of ":"
+ Colon token.Pos // position of ":"
Value Expr
}
)
type (
// An ArrayType node represents an array or slice type.
ArrayType struct {
- Lbrack token.Position // position of "["
- Len Expr // Ellipsis node for [...]T array types, nil for slice types
- Elt Expr // element type
+ Lbrack token.Pos // position of "["
+ Len Expr // Ellipsis node for [...]T array types, nil for slice types
+ Elt Expr // element type
}
// A StructType node represents a struct type.
StructType struct {
- Struct token.Position // position of "struct" keyword
- Fields *FieldList // list of field declarations
- Incomplete bool // true if (source) fields are missing in the Fields list
+ Struct token.Pos // position of "struct" keyword
+ Fields *FieldList // list of field declarations
+ Incomplete bool // true if (source) fields are missing in the Fields list
}
// Pointer types are represented via StarExpr nodes.
// A FuncType node represents a function type.
FuncType struct {
- Func token.Position // position of "func" keyword
- Params *FieldList // (incoming) parameters
- Results *FieldList // (outgoing) results
+ Func token.Pos // position of "func" keyword
+ Params *FieldList // (incoming) parameters
+ Results *FieldList // (outgoing) results
}
// An InterfaceType node represents an interface type.
InterfaceType struct {
- Interface token.Position // position of "interface" keyword
- Methods *FieldList // list of methods
- Incomplete bool // true if (source) methods are missing in the Methods list
+ Interface token.Pos // position of "interface" keyword
+ Methods *FieldList // list of methods
+ Incomplete bool // true if (source) methods are missing in the Methods list
}
// A MapType node represents a map type.
MapType struct {
- Map token.Position // position of "map" keyword
+ Map token.Pos // position of "map" keyword
Key Expr
Value Expr
}
// A ChanType node represents a channel type.
ChanType struct {
- Begin token.Position // position of "chan" keyword or "<-" (whichever comes first)
- Dir ChanDir // channel direction
- Value Expr // value type
+ Begin token.Pos // position of "chan" keyword or "<-" (whichever comes first)
+ Dir ChanDir // channel direction
+ Value Expr // value type
}
)
// Pos() implementations for expression/type nodes.
//
-func (x *BadExpr) Pos() token.Position { return x.Begin }
-func (x *Ident) Pos() token.Position { return x.NamePos }
-func (x *Ellipsis) Pos() token.Position { return x.Ellipsis }
-func (x *BasicLit) Pos() token.Position { return x.ValuePos }
-func (x *FuncLit) Pos() token.Position { return x.Type.Pos() }
-func (x *CompositeLit) Pos() token.Position {
+func (x *BadExpr) Pos() token.Pos { return x.Begin }
+func (x *Ident) Pos() token.Pos { return x.NamePos }
+func (x *Ellipsis) Pos() token.Pos { return x.Ellipsis }
+func (x *BasicLit) Pos() token.Pos { return x.ValuePos }
+func (x *FuncLit) Pos() token.Pos { return x.Type.Pos() }
+func (x *CompositeLit) Pos() token.Pos {
if x.Type != nil {
return x.Type.Pos()
}
return x.Lbrace
}
-func (x *ParenExpr) Pos() token.Position { return x.Lparen }
-func (x *SelectorExpr) Pos() token.Position { return x.X.Pos() }
-func (x *IndexExpr) Pos() token.Position { return x.X.Pos() }
-func (x *SliceExpr) Pos() token.Position { return x.X.Pos() }
-func (x *TypeAssertExpr) Pos() token.Position { return x.X.Pos() }
-func (x *CallExpr) Pos() token.Position { return x.Fun.Pos() }
-func (x *StarExpr) Pos() token.Position { return x.Star }
-func (x *UnaryExpr) Pos() token.Position { return x.OpPos }
-func (x *BinaryExpr) Pos() token.Position { return x.X.Pos() }
-func (x *KeyValueExpr) Pos() token.Position { return x.Key.Pos() }
-func (x *ArrayType) Pos() token.Position { return x.Lbrack }
-func (x *StructType) Pos() token.Position { return x.Struct }
-func (x *FuncType) Pos() token.Position { return x.Func }
-func (x *InterfaceType) Pos() token.Position { return x.Interface }
-func (x *MapType) Pos() token.Position { return x.Map }
-func (x *ChanType) Pos() token.Position { return x.Begin }
+func (x *ParenExpr) Pos() token.Pos { return x.Lparen }
+func (x *SelectorExpr) Pos() token.Pos { return x.X.Pos() }
+func (x *IndexExpr) Pos() token.Pos { return x.X.Pos() }
+func (x *SliceExpr) Pos() token.Pos { return x.X.Pos() }
+func (x *TypeAssertExpr) Pos() token.Pos { return x.X.Pos() }
+func (x *CallExpr) Pos() token.Pos { return x.Fun.Pos() }
+func (x *StarExpr) Pos() token.Pos { return x.Star }
+func (x *UnaryExpr) Pos() token.Pos { return x.OpPos }
+func (x *BinaryExpr) Pos() token.Pos { return x.X.Pos() }
+func (x *KeyValueExpr) Pos() token.Pos { return x.Key.Pos() }
+func (x *ArrayType) Pos() token.Pos { return x.Lbrack }
+func (x *StructType) Pos() token.Pos { return x.Struct }
+func (x *FuncType) Pos() token.Pos { return x.Func }
+func (x *InterfaceType) Pos() token.Pos { return x.Interface }
+func (x *MapType) Pos() token.Pos { return x.Map }
+func (x *ChanType) Pos() token.Pos { return x.Begin }
// exprNode() ensures that only expression/type nodes can be
// ----------------------------------------------------------------------------
// Convenience functions for Idents
-var noPos token.Position
+var noPos token.Pos
// NewIdent creates a new Ident without position.
// Useful for ASTs generated by code other than the Go parser.
// created.
//
BadStmt struct {
- Begin token.Position // beginning position of bad statement
+ Begin token.Pos // beginning position of bad statement
}
// A DeclStmt node represents a declaration in a statement list.
// of the immediately preceeding semicolon.
//
EmptyStmt struct {
- Semicolon token.Position // position of preceeding ";"
+ Semicolon token.Pos // position of preceeding ";"
}
// A LabeledStmt node represents a labeled statement.
//
AssignStmt struct {
Lhs []Expr
- TokPos token.Position // position of Tok
- Tok token.Token // assignment token, DEFINE
+ TokPos token.Pos // position of Tok
+ Tok token.Token // assignment token, DEFINE
Rhs []Expr
}
// A GoStmt node represents a go statement.
GoStmt struct {
- Go token.Position // position of "go" keyword
+ Go token.Pos // position of "go" keyword
Call *CallExpr
}
// A DeferStmt node represents a defer statement.
DeferStmt struct {
- Defer token.Position // position of "defer" keyword
+ Defer token.Pos // position of "defer" keyword
Call *CallExpr
}
// A ReturnStmt node represents a return statement.
ReturnStmt struct {
- Return token.Position // position of "return" keyword
+ Return token.Pos // position of "return" keyword
Results []Expr
}
// or fallthrough statement.
//
BranchStmt struct {
- TokPos token.Position // position of Tok
- Tok token.Token // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH)
+ TokPos token.Pos // position of Tok
+ Tok token.Token // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH)
Label *Ident
}
// A BlockStmt node represents a braced statement list.
BlockStmt struct {
- Lbrace token.Position // position of "{"
+ Lbrace token.Pos // position of "{"
List []Stmt
- Rbrace token.Position // position of "}"
+ Rbrace token.Pos // position of "}"
}
// An IfStmt node represents an if statement.
IfStmt struct {
- If token.Position // position of "if" keyword
+ If token.Pos // position of "if" keyword
Init Stmt
Cond Expr
Body *BlockStmt
// A CaseClause represents a case of an expression switch statement.
CaseClause struct {
- Case token.Position // position of "case" or "default" keyword
- Values []Expr // nil means default case
- Colon token.Position // position of ":"
- Body []Stmt // statement list; or nil
+ Case token.Pos // position of "case" or "default" keyword
+ Values []Expr // nil means default case
+ Colon token.Pos // position of ":"
+ Body []Stmt // statement list; or nil
}
// A SwitchStmt node represents an expression switch statement.
SwitchStmt struct {
- Switch token.Position // position of "switch" keyword
+ Switch token.Pos // position of "switch" keyword
Init Stmt
Tag Expr
Body *BlockStmt // CaseClauses only
// A TypeCaseClause represents a case of a type switch statement.
TypeCaseClause struct {
- Case token.Position // position of "case" or "default" keyword
- Types []Expr // nil means default case
- Colon token.Position // position of ":"
- Body []Stmt // statement list; or nil
+ Case token.Pos // position of "case" or "default" keyword
+ Types []Expr // nil means default case
+ Colon token.Pos // position of ":"
+ Body []Stmt // statement list; or nil
}
// An TypeSwitchStmt node represents a type switch statement.
TypeSwitchStmt struct {
- Switch token.Position // position of "switch" keyword
+ Switch token.Pos // position of "switch" keyword
Init Stmt
Assign Stmt // x := y.(type)
Body *BlockStmt // TypeCaseClauses only
// A CommClause node represents a case of a select statement.
CommClause struct {
- Case token.Position // position of "case" or "default" keyword
- Tok token.Token // ASSIGN or DEFINE (valid only if Lhs != nil)
- Lhs, Rhs Expr // Rhs == nil means default case
- Colon token.Position // position of ":"
- Body []Stmt // statement list; or nil
+ Case token.Pos // position of "case" or "default" keyword
+ Tok token.Token // ASSIGN or DEFINE (valid only if Lhs != nil)
+ Lhs, Rhs Expr // Rhs == nil means default case
+ Colon token.Pos // position of ":"
+ Body []Stmt // statement list; or nil
}
// An SelectStmt node represents a select statement.
SelectStmt struct {
- Select token.Position // position of "select" keyword
- Body *BlockStmt // CommClauses only
+ Select token.Pos // position of "select" keyword
+ Body *BlockStmt // CommClauses only
}
// A ForStmt represents a for statement.
ForStmt struct {
- For token.Position // position of "for" keyword
+ For token.Pos // position of "for" keyword
Init Stmt
Cond Expr
Post Stmt
// A RangeStmt represents a for statement with a range clause.
RangeStmt struct {
- For token.Position // position of "for" keyword
- Key, Value Expr // Value may be nil
- TokPos token.Position // position of Tok
- Tok token.Token // ASSIGN, DEFINE
- X Expr // value to range over
+ For token.Pos // position of "for" keyword
+ Key, Value Expr // Value may be nil
+ TokPos token.Pos // position of Tok
+ Tok token.Token // ASSIGN, DEFINE
+ X Expr // value to range over
Body *BlockStmt
}
)
// Pos() implementations for statement nodes.
//
-func (s *BadStmt) Pos() token.Position { return s.Begin }
-func (s *DeclStmt) Pos() token.Position { return s.Decl.Pos() }
-func (s *EmptyStmt) Pos() token.Position { return s.Semicolon }
-func (s *LabeledStmt) Pos() token.Position { return s.Label.Pos() }
-func (s *ExprStmt) Pos() token.Position { return s.X.Pos() }
-func (s *IncDecStmt) Pos() token.Position { return s.X.Pos() }
-func (s *AssignStmt) Pos() token.Position { return s.Lhs[0].Pos() }
-func (s *GoStmt) Pos() token.Position { return s.Go }
-func (s *DeferStmt) Pos() token.Position { return s.Defer }
-func (s *ReturnStmt) Pos() token.Position { return s.Return }
-func (s *BranchStmt) Pos() token.Position { return s.TokPos }
-func (s *BlockStmt) Pos() token.Position { return s.Lbrace }
-func (s *IfStmt) Pos() token.Position { return s.If }
-func (s *CaseClause) Pos() token.Position { return s.Case }
-func (s *SwitchStmt) Pos() token.Position { return s.Switch }
-func (s *TypeCaseClause) Pos() token.Position { return s.Case }
-func (s *TypeSwitchStmt) Pos() token.Position { return s.Switch }
-func (s *CommClause) Pos() token.Position { return s.Case }
-func (s *SelectStmt) Pos() token.Position { return s.Select }
-func (s *ForStmt) Pos() token.Position { return s.For }
-func (s *RangeStmt) Pos() token.Position { return s.For }
+func (s *BadStmt) Pos() token.Pos { return s.Begin }
+func (s *DeclStmt) Pos() token.Pos { return s.Decl.Pos() }
+func (s *EmptyStmt) Pos() token.Pos { return s.Semicolon }
+func (s *LabeledStmt) Pos() token.Pos { return s.Label.Pos() }
+func (s *ExprStmt) Pos() token.Pos { return s.X.Pos() }
+func (s *IncDecStmt) Pos() token.Pos { return s.X.Pos() }
+func (s *AssignStmt) Pos() token.Pos { return s.Lhs[0].Pos() }
+func (s *GoStmt) Pos() token.Pos { return s.Go }
+func (s *DeferStmt) Pos() token.Pos { return s.Defer }
+func (s *ReturnStmt) Pos() token.Pos { return s.Return }
+func (s *BranchStmt) Pos() token.Pos { return s.TokPos }
+func (s *BlockStmt) Pos() token.Pos { return s.Lbrace }
+func (s *IfStmt) Pos() token.Pos { return s.If }
+func (s *CaseClause) Pos() token.Pos { return s.Case }
+func (s *SwitchStmt) Pos() token.Pos { return s.Switch }
+func (s *TypeCaseClause) Pos() token.Pos { return s.Case }
+func (s *TypeSwitchStmt) Pos() token.Pos { return s.Switch }
+func (s *CommClause) Pos() token.Pos { return s.Case }
+func (s *SelectStmt) Pos() token.Pos { return s.Select }
+func (s *ForStmt) Pos() token.Pos { return s.For }
+func (s *RangeStmt) Pos() token.Pos { return s.For }
// stmtNode() ensures that only statement nodes can be
// Pos() implementations for spec nodes.
//
-func (s *ImportSpec) Pos() token.Position {
+func (s *ImportSpec) Pos() token.Pos {
if s.Name != nil {
return s.Name.Pos()
}
return s.Path.Pos()
}
-func (s *ValueSpec) Pos() token.Position { return s.Names[0].Pos() }
-func (s *TypeSpec) Pos() token.Position { return s.Name.Pos() }
+func (s *ValueSpec) Pos() token.Pos { return s.Names[0].Pos() }
+func (s *TypeSpec) Pos() token.Pos { return s.Name.Pos() }
// specNode() ensures that only spec nodes can be
// created.
//
BadDecl struct {
- Begin token.Position // beginning position of bad declaration
+ Begin token.Pos // beginning position of bad declaration
}
// A GenDecl node (generic declaration node) represents an import,
// token.VAR *ValueSpec
//
GenDecl struct {
- Doc *CommentGroup // associated documentation; or nil
- TokPos token.Position // position of Tok
- Tok token.Token // IMPORT, CONST, TYPE, VAR
- Lparen token.Position // position of '(', if any
+ Doc *CommentGroup // associated documentation; or nil
+ TokPos token.Pos // position of Tok
+ Tok token.Token // IMPORT, CONST, TYPE, VAR
+ Lparen token.Pos // position of '(', if any
Specs []Spec
- Rparen token.Position // position of ')', if any
+ Rparen token.Pos // position of ')', if any
}
// A FuncDecl node represents a function declaration.
// Pos implementations for declaration nodes.
//
-func (d *BadDecl) Pos() token.Position { return d.Begin }
-func (d *GenDecl) Pos() token.Position { return d.TokPos }
-func (d *FuncDecl) Pos() token.Position { return d.Type.Pos() }
+func (d *BadDecl) Pos() token.Pos { return d.Begin }
+func (d *GenDecl) Pos() token.Pos { return d.TokPos }
+func (d *FuncDecl) Pos() token.Pos { return d.Type.Pos() }
// declNode() ensures that only declaration nodes can be
//
type File struct {
Doc *CommentGroup // associated documentation; or nil
- Package token.Position // position of "package" keyword
+ Package token.Pos // position of "package" keyword
Name *Ident // package name
Decls []Decl // top-level declarations
Comments []*CommentGroup // list of all comments in the source file
}
-func (f *File) Pos() token.Position { return f.Package }
+func (f *File) Pos() token.Pos { return f.Package }
// A Package node represents a set of source files
var separator = &Comment{noPos, []byte("//")}
-// lineAfterComment computes the position of the beginning
-// of the line immediately following a comment.
-func lineAfterComment(c *Comment) token.Position {
- pos := c.Pos()
- line := pos.Line
- text := c.Text
- if text[1] == '*' {
- /*-style comment - determine endline */
- for _, ch := range text {
- if ch == '\n' {
- line++
- }
- }
- }
- pos.Offset += len(text) + 1 // +1 for newline
- pos.Line = line + 1 // line after comment
- pos.Column = 1 // beginning of line
- return pos
-}
-
-
// MergePackageFiles creates a file AST by merging the ASTs of the
// files belonging to a package. The mode flags control merging behavior.
//
// a package comment; but it's better to collect extra comments
// than drop them on the floor.
var doc *CommentGroup
- var pos token.Position
+ var pos token.Pos
if ndocs > 0 {
list := make([]*Comment, ndocs-1) // -1: no separator before first group
i := 0
list[i] = c
i++
}
- end := lineAfterComment(f.Doc.List[len(f.Doc.List)-1])
- if end.Offset > pos.Offset {
- // Keep the maximum end position as
- // position for the package clause.
- pos = end
+ if f.Package > pos {
+ // Keep the maximum package clause position as
+ // position for the package clause of the merged
+ // files.
+ pos = f.Package
}
}
}
n2 := len(comments.List)
list := make([]*ast.Comment, n1+1+n2) // + 1 for separator line
copy(list, doc.doc.List)
- list[n1] = &ast.Comment{token.Position{}, []byte("//")} // separator line
+ list[n1] = &ast.Comment{token.NoPos, []byte("//")} // separator line
copy(list[n1+1:], comments.List)
doc.doc = &ast.CommentGroup{list}
}
doc.addValue(d)
case token.TYPE:
// types are handled individually
- var noPos token.Position
for _, spec := range d.Specs {
// make a (fake) GenDecl node for this TypeSpec
// (we need to do this here - as opposed to just
// makeTypeDocs below). Simpler data structures, but
// would lose GenDecl documentation if the TypeSpec
// has documentation as well.
- doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, noPos, []ast.Spec{spec}, noPos})
+ doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, token.NoPos, []ast.Spec{spec}, token.NoPos})
// A new GenDecl node is created, no need to nil out d.Doc.
}
}
// ParseExpr parses a Go expression and returns the corresponding
-// AST node. The filename and src arguments have the same interpretation
+// AST node. The fset, filename, and src arguments have the same interpretation
// as for ParseFile. If there is an error, the result expression
// may be nil or contain a partial AST.
//
-func ParseExpr(filename string, src interface{}) (ast.Expr, os.Error) {
+func ParseExpr(fset *token.FileSet, filename string, src interface{}) (ast.Expr, os.Error) {
data, err := readSource(filename, src)
if err != nil {
return nil, err
}
var p parser
- p.init(filename, data, 0)
+ p.init(fset, filename, data, 0)
x := p.parseExpr()
if p.tok == token.SEMICOLON {
p.next() // consume automatically inserted semicolon, if any
// ParseStmtList parses a list of Go statements and returns the list
-// of corresponding AST nodes. The filename and src arguments have the same
+// of corresponding AST nodes. The fset, filename, and src arguments have the same
// interpretation as for ParseFile. If there is an error, the node
// list may be nil or contain partial ASTs.
//
-func ParseStmtList(filename string, src interface{}) ([]ast.Stmt, os.Error) {
+func ParseStmtList(fset *token.FileSet, filename string, src interface{}) ([]ast.Stmt, os.Error) {
data, err := readSource(filename, src)
if err != nil {
return nil, err
}
var p parser
- p.init(filename, data, 0)
+ p.init(fset, filename, data, 0)
return p.parseStmtList(), p.parseEOF()
}
// ParseDeclList parses a list of Go declarations and returns the list
-// of corresponding AST nodes. The filename and src arguments have the same
+// of corresponding AST nodes. The fset, filename, and src arguments have the same
// interpretation as for ParseFile. If there is an error, the node
// list may be nil or contain partial ASTs.
//
-func ParseDeclList(filename string, src interface{}) ([]ast.Decl, os.Error) {
+func ParseDeclList(fset *token.FileSet, filename string, src interface{}) ([]ast.Decl, os.Error) {
data, err := readSource(filename, src)
if err != nil {
return nil, err
}
var p parser
- p.init(filename, data, 0)
+ p.init(fset, filename, data, 0)
return p.parseDeclList(), p.parseEOF()
}
+// TODO(gri) Change ParseFile to Parse and improve documentation (issue 1311).
+
// ParseFile parses a Go source file and returns a File node.
//
// If src != nil, ParseFile parses the file source from src. src may
// If src == nil, ParseFile parses the file specified by filename.
//
// The mode parameter controls the amount of source text parsed and other
-// optional parser functionality.
+// optional parser functionality. Position information is recorded in the
+// file set fset.
//
// If the source couldn't be read, the returned AST is nil and the error
// indicates the specific failure. If the source was read but syntax
// representing the fragments of erroneous source code). Multiple errors
// are returned via a scanner.ErrorList which is sorted by file position.
//
-func ParseFile(filename string, src interface{}, mode uint) (*ast.File, os.Error) {
+func ParseFile(fset *token.FileSet, filename string, src interface{}, mode uint) (*ast.File, os.Error) {
data, err := readSource(filename, src)
if err != nil {
return nil, err
}
var p parser
- p.init(filename, data, mode)
+ p.init(fset, filename, data, mode)
return p.parseFile(), p.GetError(scanner.NoMultiples) // parseFile() reads to EOF
}
// ParseFiles calls ParseFile for each file in the filenames list and returns
// a map of package name -> package AST with all the packages found. The mode
-// bits are passed to ParseFile unchanged.
+// bits are passed to ParseFile unchanged. Position information is recorded
+// in the file set fset.
//
// Files with parse errors are ignored. In this case the map of packages may
// be incomplete (missing packages and/or incomplete packages) and the first
// error encountered is returned.
//
-func ParseFiles(filenames []string, mode uint) (pkgs map[string]*ast.Package, first os.Error) {
+func ParseFiles(fset *token.FileSet, filenames []string, mode uint) (pkgs map[string]*ast.Package, first os.Error) {
pkgs = make(map[string]*ast.Package)
for _, filename := range filenames {
- if src, err := ParseFile(filename, nil, mode); err == nil {
+ if src, err := ParseFile(fset, filename, nil, mode); err == nil {
name := src.Name.Name
pkg, found := pkgs[name]
if !found {
// ParseDir calls ParseFile for the files in the directory specified by path and
// returns a map of package name -> package AST with all the packages found. If
// filter != nil, only the files with os.FileInfo entries passing through the filter
-// are considered. The mode bits are passed to ParseFile unchanged.
+// are considered. The mode bits are passed to ParseFile unchanged. Position
+// information is recorded in the file set fset.
//
// If the directory couldn't be read, a nil map and the respective error are
// returned. If a parse error occured, a non-nil but incomplete map and the
// error are returned.
//
-func ParseDir(path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) {
+func ParseDir(fset *token.FileSet, path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) {
fd, err := os.Open(path, os.O_RDONLY, 0)
if err != nil {
return nil, err
}
filenames = filenames[0:n]
- return ParseFiles(filenames, mode)
+ return ParseFiles(fset, filenames, mode)
}
// The parser structure holds the parser's internal state.
type parser struct {
+ file *token.File
scanner.ErrorVector
scanner scanner.Scanner
lineComment *ast.CommentGroup // the last line comment
// Next token
- pos token.Position // token position
- tok token.Token // one token look-ahead
- lit []byte // token literal
+ pos token.Pos // token position
+ tok token.Token // one token look-ahead
+ lit []byte // token literal
// Non-syntactic parser control
exprLev int // < 0: in control clause, >= 0: in expression
}
-func (p *parser) init(filename string, src []byte, mode uint) {
- p.scanner.Init(filename, src, p, scannerMode(mode))
+func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) {
+ p.file = p.scanner.Init(fset, filename, src, p, scannerMode(mode))
p.mode = mode
p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently)
p.next()
const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +
". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
const n = uint(len(dots))
- fmt.Printf("%5d:%3d: ", p.pos.Line, p.pos.Column)
+ pos := p.file.Position(p.pos)
+ fmt.Printf("%5d:%3d: ", pos.Line, pos.Column)
i := 2 * p.indent
for ; i > n; i -= n {
fmt.Print(dots)
func (p *parser) next0() {
// Because of one-token look-ahead, print the previous token
// when tracing as it provides a more readable output. The
- // very first token (p.pos.Line == 0) is not initialized (it
- // is token.ILLEGAL), so don't print it .
- if p.trace && p.pos.Line > 0 {
+ // very first token (!p.pos.IsValid()) is not initialized
+ // (it is token.ILLEGAL), so don't print it .
+ if p.trace && p.pos.IsValid() {
s := p.tok.String()
switch {
case p.tok.IsLiteral():
func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
// /*-style comments may end on a different line than where they start.
// Scan the comment for '\n' chars and adjust endline accordingly.
- endline = p.pos.Line
+ endline = p.file.Line(p.pos)
if p.lit[1] == '*' {
for _, b := range p.lit {
if b == '\n' {
//
func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) {
var list []*ast.Comment
- endline = p.pos.Line
- for p.tok == token.COMMENT && endline+1 >= p.pos.Line {
+ endline = p.file.Line(p.pos)
+ for p.tok == token.COMMENT && endline+1 >= p.file.Line(p.pos) {
var comment *ast.Comment
comment, endline = p.consumeComment()
list = append(list, comment)
func (p *parser) next() {
p.leadComment = nil
p.lineComment = nil
- line := p.pos.Line // current line
+ line := p.file.Line(p.pos) // current line
p.next0()
if p.tok == token.COMMENT {
var comment *ast.CommentGroup
var endline int
- if p.pos.Line == line {
+ if p.file.Line(p.pos) == line {
// The comment is on same line as previous token; it
// cannot be a lead comment but may be a line comment.
comment, endline = p.consumeCommentGroup()
- if p.pos.Line != endline {
+ if p.file.Line(p.pos) != endline {
// The next token is on a different line, thus
// the last comment group is a line comment.
p.lineComment = comment
comment, endline = p.consumeCommentGroup()
}
- if endline+1 == p.pos.Line {
+ if endline+1 == p.file.Line(p.pos) {
// The next token is following on the line immediately after the
// comment group, thus the last comment group is a lead comment.
p.leadComment = comment
}
-func (p *parser) errorExpected(pos token.Position, msg string) {
+func (p *parser) error(pos token.Pos, msg string) {
+ p.Error(p.file.Position(pos), msg)
+}
+
+
+func (p *parser) errorExpected(pos token.Pos, msg string) {
msg = "expected " + msg
- if pos.Offset == p.pos.Offset {
+ if pos == p.pos {
// the error happened at the current position;
// make the error message more specific
if p.tok == token.SEMICOLON && p.lit[0] == '\n' {
}
}
}
- p.Error(pos, msg)
+ p.error(pos, msg)
}
-func (p *parser) expect(tok token.Token) token.Position {
+func (p *parser) expect(tok token.Token) token.Pos {
pos := p.pos
if p.tok != tok {
p.errorExpected(pos, "'"+tok.String()+"'")
p.next()
typ := p.tryType() // don't use parseType so we can provide better error message
if typ == nil {
- p.Error(pos, "'...' parameter is missing type")
+ p.error(pos, "'...' parameter is missing type")
typ = &ast.BadExpr{pos}
}
if p.tok != token.RPAREN {
- p.Error(pos, "can use '...' with last parameter type only")
+ p.error(pos, "can use '...' with last parameter type only")
}
return &ast.Ellipsis{pos, typ}
}
// method
idents = []*ast.Ident{ident}
params, results := p.parseSignature()
- typ = &ast.FuncType{noPos, params, results}
+ typ = &ast.FuncType{token.NoPos, params, results}
} else {
// embedded interface
typ = x
lparen := p.expect(token.LPAREN)
p.exprLev++
var list []ast.Expr
- var ellipsis token.Position
+ var ellipsis token.Pos
for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() {
list = append(list, p.parseExpr())
if p.tok == token.ELLIPSIS {
}
case *ast.ArrayType:
if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis {
- p.Error(len.Pos(), "expected array length, found '...'")
+ p.error(len.Pos(), "expected array length, found '...'")
x = &ast.BadExpr{x.Pos()}
}
}
return &ast.LabeledStmt{label, p.parseStmt()}
}
}
- p.Error(x[0].Pos(), "illegal label declaration")
+ p.error(x[0].Pos(), "illegal label declaration")
return &ast.BadStmt{x[0].Pos()}
case
}
if len(x) > 1 {
- p.Error(x[0].Pos(), "only one expression allowed")
+ p.error(x[0].Pos(), "only one expression allowed")
// continue with first expression
}
if es, isExpr := s.(*ast.ExprStmt); isExpr {
return p.checkExpr(es.X)
}
- p.Error(s.Pos(), "expected condition, found simple statement")
+ p.error(s.Pos(), "expected condition, found simple statement")
return &ast.BadExpr{s.Pos()}
}
doc := p.leadComment
pos := p.expect(keyword)
- var lparen, rparen token.Position
+ var lparen, rparen token.Pos
var list []ast.Spec
if p.tok == token.LPAREN {
lparen = p.pos
// must have exactly one receiver
if par.NumFields() != 1 {
p.errorExpected(pos, "exactly one receiver")
- par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{noPos}}}
+ par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{token.NoPos}}}
return par
}
package parser
import (
+ "go/token"
"os"
"testing"
)
+var fset = token.NewFileSet()
+
var illegalInputs = []interface{}{
nil,
3.14,
func TestParseIllegalInputs(t *testing.T) {
for _, src := range illegalInputs {
- _, err := ParseFile("", src, 0)
+ _, err := ParseFile(fset, "", src, 0)
if err == nil {
t.Errorf("ParseFile(%v) should have failed", src)
}
func TestParseValidPrograms(t *testing.T) {
for _, src := range validPrograms {
- _, err := ParseFile("", src, 0)
+ _, err := ParseFile(fset, "", src, 0)
if err != nil {
t.Errorf("ParseFile(%q): %v", src, err)
}
func TestParse3(t *testing.T) {
for _, filename := range validFiles {
- _, err := ParseFile(filename, nil, 0)
+ _, err := ParseFile(fset, filename, nil, 0)
if err != nil {
t.Errorf("ParseFile(%s): %v", filename, err)
}
func TestParse4(t *testing.T) {
path := "."
- pkgs, err := ParseDir(path, dirFilter, 0)
+ pkgs, err := ParseDir(fset, path, dirFilter, 0)
if err != nil {
t.Fatalf("ParseDir(%s): %v", path, err)
}
// for some reason there are pending comments; this
// should never happen - handle gracefully and flush
// all comments up to g, ignore anything after that
- p.flush(g.List[0].Pos(), token.ILLEGAL)
+ p.flush(p.fset.Position(g.List[0].Pos()), token.ILLEGAL)
}
p.comments[0] = g
p.cindex = 0
if !indent {
mode |= noIndent
}
- p.exprList(noPos, xlist, 1, mode, multiLine, noPos)
+ p.exprList(token.NoPos, xlist, 1, mode, multiLine, token.NoPos)
}
// TODO(gri) Consider rewriting this to be independent of []ast.Expr
// so that we can use the algorithm for any kind of list
// (e.g., pass list via a channel over which to range).
-func (p *printer) exprList(prev token.Position, list []ast.Expr, depth int, mode exprListMode, multiLine *bool, next token.Position) {
+func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exprListMode, multiLine *bool, next0 token.Pos) {
if len(list) == 0 {
return
}
p.print(blank)
}
- line := list[0].Pos().Line
+ prev := p.fset.Position(prev0)
+ next := p.fset.Position(next0)
+ line := p.fset.Position(list[0].Pos()).Line
endLine := next.Line
if endLine == 0 {
// TODO(gri): endLine may be incorrect as it is really the beginning
// of the last list entry. There may be only one, very long
// entry in which case line == endLine.
- endLine = list[len(list)-1].Pos().Line
+ endLine = p.fset.Position(list[len(list)-1].Pos()).Line
}
if prev.IsValid() && prev.Line == line && line == endLine {
// print all list elements
for i, x := range list {
prevLine := line
- line = x.Pos().Line
+ line = p.fset.Position(x.Pos()).Line
// determine if the next linebreak, if any, needs to use formfeed:
// in general, use the entire node size to make the decision; for
if i > 0 {
p.print(token.COMMA)
if len(par.Names) > 0 {
- line = par.Names[0].Pos().Line
+ line = p.fset.Position(par.Names[0].Pos()).Line
} else {
- line = par.Type.Pos().Line
+ line = p.fset.Position(par.Type.Pos()).Line
}
if 0 < prevLine && prevLine < line && p.linebreak(line, 0, ignore, true) {
*multiLine = true
p.print(blank)
}
p.expr(par.Type, multiLine)
- prevLine = par.Type.Pos().Line
+ prevLine = p.fset.Position(par.Type.Pos()).Line
}
}
p.print(fields.Closing, token.RPAREN)
func (p *printer) setLineComment(text string) {
- p.setComment(&ast.CommentGroup{[]*ast.Comment{&ast.Comment{noPos, []byte(text)}}})
+ p.setComment(&ast.CommentGroup{[]*ast.Comment{&ast.Comment{token.NoPos, []byte(text)}}})
}
list := fields.List
rbrace := fields.Closing
- if !isIncomplete && !p.commentBefore(rbrace) {
+ if !isIncomplete && !p.commentBefore(p.fset.Position(rbrace)) {
// possibly a one-line struct/interface
if len(list) == 0 {
// no blank between keyword and {} in this case
var ml bool
for i, f := range list {
if i > 0 {
- p.linebreak(f.Pos().Line, 1, ignore, ml)
+ p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
}
ml = false
extraTabs := 0
if len(list) > 0 {
p.print(formfeed)
}
- p.flush(rbrace, token.RBRACE) // make sure we don't loose the last line comment
+ p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't loose the last line comment
p.setLineComment("// contains unexported fields")
}
var ml bool
for i, f := range list {
if i > 0 {
- p.linebreak(f.Pos().Line, 1, ignore, ml)
+ p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
}
ml = false
p.setComment(f.Doc)
if len(list) > 0 {
p.print(formfeed)
}
- p.flush(rbrace, token.RBRACE) // make sure we don't loose the last line comment
+ p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't loose the last line comment
p.setLineComment("// contains unexported methods")
}
p.print(blank)
}
xline := p.pos.Line // before the operator (it may be on the next line!)
- yline := x.Y.Pos().Line
+ yline := p.fset.Position(x.Y.Pos()).Line
p.print(x.OpPos, x.Op)
if xline != yline && xline > 0 && yline > 0 {
// at least one line break, but respect an extra empty line
case *ast.FuncLit:
p.expr(x.Type, multiLine)
- p.funcBody(x.Body, distance(x.Type.Pos(), p.pos), true, multiLine)
+ p.funcBody(x.Body, p.distance(x.Type.Pos(), p.pos), true, multiLine)
case *ast.ParenExpr:
if _, hasParens := x.X.(*ast.ParenExpr); hasParens {
case *ast.SelectorExpr:
parts := selectorExprList(expr)
- p.exprList(noPos, parts, depth, periodSep, multiLine, noPos)
+ p.exprList(token.NoPos, parts, depth, periodSep, multiLine, token.NoPos)
case *ast.TypeAssertExpr:
p.expr1(x.X, token.HighestPrec, depth, 0, multiLine)
for i, s := range list {
// _indent == 0 only for lists of switch/select case clauses;
// in those cases each clause is a new section
- p.linebreak(s.Pos().Line, 1, ignore, i == 0 || _indent == 0 || multiLine)
+ p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, i == 0 || _indent == 0 || multiLine)
multiLine = false
p.stmt(s, nextIsRBrace && i == len(list)-1, &multiLine)
}
func (p *printer) block(s *ast.BlockStmt, indent int) {
p.print(s.Pos(), token.LBRACE)
p.stmtList(s.List, indent, true)
- p.linebreak(s.Rbrace.Line, 1, ignore, true)
+ p.linebreak(p.fset.Position(s.Rbrace).Line, 1, ignore, true)
p.print(s.Rbrace, token.RBRACE)
}
break
}
} else {
- p.linebreak(s.Stmt.Pos().Line, 1, ignore, true)
+ p.linebreak(p.fset.Position(s.Stmt.Pos()).Line, 1, ignore, true)
}
p.stmt(s.Stmt, nextIsRBrace, multiLine)
}
p.exprList(s.Pos(), s.Lhs, depth, commaSep, multiLine, s.TokPos)
p.print(blank, s.TokPos, s.Tok)
- p.exprList(s.TokPos, s.Rhs, depth, blankStart|commaSep, multiLine, noPos)
+ p.exprList(s.TokPos, s.Rhs, depth, blankStart|commaSep, multiLine, token.NoPos)
case *ast.GoStmt:
p.print(token.GO, blank)
case *ast.ReturnStmt:
p.print(token.RETURN)
if s.Results != nil {
- p.exprList(s.Pos(), s.Results, 1, blankStart|commaSep, multiLine, noPos)
+ p.exprList(s.Pos(), s.Results, 1, blankStart|commaSep, multiLine, token.NoPos)
}
case *ast.BranchStmt:
}
if s.Values != nil {
p.print(blank, token.ASSIGN)
- p.exprList(noPos, s.Values, 1, blankStart|commaSep, multiLine, noPos)
+ p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
}
p.setComment(s.Comment)
}
if s.Values != nil {
p.print(vtab, token.ASSIGN)
- p.exprList(noPos, s.Values, 1, blankStart|commaSep, multiLine, noPos)
+ p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
extraTabs--
}
if s.Comment != nil {
var ml bool
for i, s := range d.Specs {
if i > 0 {
- p.linebreak(s.Pos().Line, 1, ignore, ml)
+ p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml)
}
ml = false
p.spec(s, len(d.Specs), false, &ml)
// in RawFormat
cfg := Config{Mode: RawFormat}
var buf bytes.Buffer
- if _, err := cfg.Fprint(&buf, n); err != nil {
+ if _, err := cfg.Fprint(&buf, p.fset, n); err != nil {
return
}
if buf.Len() <= maxSize {
func (p *printer) isOneLineFunc(b *ast.BlockStmt, headerSize int) bool {
pos1 := b.Pos()
pos2 := b.Rbrace
- if pos1.IsValid() && pos2.IsValid() && pos1.Line != pos2.Line {
+ if pos1.IsValid() && pos2.IsValid() && p.fset.Position(pos1).Line != p.fset.Position(pos2).Line {
// opening and closing brace are on different lines - don't make it a one-liner
return false
}
- if len(b.List) > 5 || p.commentBefore(pos2) {
+ if len(b.List) > 5 || p.commentBefore(p.fset.Position(pos2)) {
// too many statements or there is a comment inside - don't make it a one-liner
return false
}
// distance returns the column difference between from and to if both
// are on the same line; if they are on different lines (or unknown)
// the result is infinity.
-func distance(from, to token.Position) int {
+func (p *printer) distance(from0 token.Pos, to token.Position) int {
+ from := p.fset.Position(from0)
if from.IsValid() && to.IsValid() && from.Line == to.Line {
return to.Column - from.Column
}
}
p.expr(d.Name, multiLine)
p.signature(d.Type.Params, d.Type.Results, multiLine)
- p.funcBody(d.Body, distance(d.Pos(), p.pos), false, multiLine)
+ p.funcBody(d.Body, p.distance(d.Pos(), p.pos), false, multiLine)
}
if prev != tok {
min = 2
}
- p.linebreak(d.Pos().Line, min, ignore, false)
+ p.linebreak(p.fset.Position(d.Pos()).Line, min, ignore, false)
p.decl(d, ignoreMultiLine)
}
}
// Configuration (does not change after initialization)
output io.Writer
Config
+ fset *token.FileSet
errors chan os.Error
// Current state
}
-func (p *printer) init(output io.Writer, cfg *Config) {
+func (p *printer) init(output io.Writer, cfg *Config, fset *token.FileSet) {
p.output = output
p.Config = *cfg
+ p.fset = fset
p.errors = make(chan os.Error)
p.buffer = make([]whiteSpace, 0, 16) // whitespace sequences are short
}
// shortcut common case of //-style comments
if text[1] == '/' {
- p.writeCommentLine(comment, comment.Pos(), text)
+ p.writeCommentLine(comment, p.fset.Position(comment.Pos()), text)
return
}
// write comment lines, separated by formfeed,
// without a line break after the last line
linebreak := formfeeds[0:1]
- pos := comment.Pos()
+ pos := p.fset.Position(comment.Pos())
for i, line := range lines {
if i > 0 {
p.write(linebreak)
var last *ast.Comment
for ; p.commentBefore(next); p.cindex++ {
for _, c := range p.comments[p.cindex].List {
- p.writeCommentPrefix(c.Pos(), next, last == nil, tok.IsKeyword())
+ p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last == nil, tok.IsKeyword())
p.writeComment(c)
last = c
}
}
if last != nil {
- if last.Text[1] == '*' && last.Pos().Line == next.Line {
+ if last.Text[1] == '*' && p.fset.Position(last.Pos()).Line == next.Line {
// the last comment is a /*-style comment and the next item
// follows on the same line: separate with an extra blank
p.write([]byte{' '})
data = []byte(s)
}
tok = x
- case token.Position:
+ case token.Pos:
if x.IsValid() {
- next = x // accurate position of next item
+ next = p.fset.Position(x) // accurate position of next item
}
tok = p.lastTok
default:
// before the next position in the source code.
//
func (p *printer) commentBefore(next token.Position) bool {
- return p.cindex < len(p.comments) && p.comments[p.cindex].List[0].Pos().Offset < next.Offset
+ return p.cindex < len(p.comments) && p.fset.Position(p.comments[p.cindex].List[0].Pos()).Offset < next.Offset
}
// Fprint "pretty-prints" an AST node to output and returns the number
// of bytes written and an error (if any) for a given configuration cfg.
+// Position information is interpreted relative to the file set fset.
// The node type must be *ast.File, or assignment-compatible to ast.Expr,
// ast.Decl, ast.Spec, or ast.Stmt.
//
-func (cfg *Config) Fprint(output io.Writer, node interface{}) (int, os.Error) {
+func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{}) (int, os.Error) {
// redirect output through a trimmer to eliminate trailing whitespace
// (Input to a tabwriter must be untrimmed since trailing tabs provide
// formatting information. The tabwriter could provide trimming
// setup printer and print node
var p printer
- p.init(output, cfg)
+ p.init(output, cfg, fset)
go func() {
switch n := node.(type) {
case ast.Expr:
// Fprint "pretty-prints" an AST node to output.
// It calls Config.Fprint with default settings.
//
-func Fprint(output io.Writer, node interface{}) os.Error {
- _, err := (&Config{Tabwidth: 8}).Fprint(output, node) // don't care about number of bytes written
+func Fprint(output io.Writer, fset *token.FileSet, node interface{}) os.Error {
+ _, err := (&Config{Tabwidth: 8}).Fprint(output, fset, node) // don't care about number of bytes written
return err
}
"io/ioutil"
"go/ast"
"go/parser"
+ "go/token"
"path"
"testing"
)
var update = flag.Bool("update", false, "update golden files")
+var fset = token.NewFileSet()
+
+
func lineString(text []byte, i int) string {
i0 := i
for i < len(text) && text[i] != '\n' {
func check(t *testing.T, source, golden string, mode checkMode) {
// parse source
- prog, err := parser.ParseFile(source, nil, parser.ParseComments)
+ prog, err := parser.ParseFile(fset, source, nil, parser.ParseComments)
if err != nil {
t.Error(err)
return
// format source
var buf bytes.Buffer
- if _, err := cfg.Fprint(&buf, prog); err != nil {
+ if _, err := cfg.Fprint(&buf, fset, prog); err != nil {
t.Error(err)
}
res := buf.Bytes()
//
type Scanner struct {
// immutable state
+ file *token.File // source file handle
src []byte // source
err ErrorHandler // error reporting; or nil
mode uint // scanning mode
// scanning state
- filename string // current filename; may change via //line filename:line comment
- line int // current line
- column int // current column
-
ch int // current character
offset int // character offset
rdOffset int // reading offset (position after current character)
+ lineOffset int // current line offset
insertSemi bool // insert a semicolon before next newline
// public state - ok to modify
// S.ch < 0 means end-of-file.
//
func (S *Scanner) next() {
- S.column++
if S.rdOffset < len(S.src) {
S.offset = S.rdOffset
if S.ch == '\n' {
- S.line++
- S.column = 1
+ S.lineOffset = S.offset
+ S.file.AddLine(S.offset)
}
r, w := int(S.src[S.rdOffset]), 1
switch {
case r == 0:
- S.error("illegal character NUL")
+ S.error(S.offset, "illegal character NUL")
case r >= 0x80:
// not ASCII
r, w = utf8.DecodeRune(S.src[S.rdOffset:])
if r == utf8.RuneError && w == 1 {
- S.error("illegal UTF-8 encoding")
+ S.error(S.offset, "illegal UTF-8 encoding")
}
}
S.rdOffset += w
} else {
S.offset = len(S.src)
if S.ch == '\n' {
- S.column = 1
+ S.lineOffset = S.offset
+ S.file.AddLine(S.offset)
}
S.ch = -1 // eof
}
InsertSemis // automatically insert semicolons
)
+// TODO(gri) Would it be better to simply provide *token.File to Init
+// instead of fset, and filename, and then return the file?
+// It could cause an error/panic if the provided file.Size()
+// doesn't match len(src).
-// Init prepares the scanner S to tokenize the text src. Calls to Scan
-// will use the error handler err if they encounter a syntax error and
-// err is not nil. Also, for each error encountered, the Scanner field
-// ErrorCount is incremented by one. The filename parameter is used as
-// filename in the token.Position returned by Scan for each token. The
-// mode parameter determines how comments and illegal characters are
-// handled.
+// Init prepares the scanner S to tokenize the text src. It sets the
+// scanner at the beginning of the source text, adds a new file with
+// the given filename to the file set fset, and returns that file.
+//
+// Calls to Scan will use the error handler err if they encounter a
+// syntax error and err is not nil. Also, for each error encountered,
+// the Scanner field ErrorCount is incremented by one. The mode parameter
+// determines how comments, illegal characters, and semicolons are handled.
//
-func (S *Scanner) Init(filename string, src []byte, err ErrorHandler, mode uint) {
+func (S *Scanner) Init(fset *token.FileSet, filename string, src []byte, err ErrorHandler, mode uint) *token.File {
// Explicitly initialize all fields since a scanner may be reused.
+ S.file = fset.AddFile(filename, fset.Base(), len(src))
S.src = src
S.err = err
S.mode = mode
- S.filename = filename
- S.line = 1
- S.column = 0
-
S.ch = ' '
S.offset = 0
S.rdOffset = 0
+ S.lineOffset = 0
S.insertSemi = false
S.ErrorCount = 0
S.next()
+
+ return S.file
}
}
-func (S *Scanner) error(msg string) {
- S.errorAt(token.Position{S.filename, S.offset, S.line, S.column}, msg)
-}
-
-
-func (S *Scanner) errorAt(pos token.Position, msg string) {
+func (S *Scanner) error(offs int, msg string) {
if S.err != nil {
- S.err.Error(pos, msg)
+ S.err.Error(S.file.Position(S.file.Pos(offs)), msg)
}
S.ErrorCount++
}
if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 {
// valid //line filename:line comment;
// update scanner position
- S.filename = string(text[len(prefix):i])
- S.line = line - 1 // -1 since the '\n' has not been consumed yet
+ S.file.AddLineInfo(S.lineOffset, string(text[len(prefix):i]), line-1) // -1 since comment applies to next line
}
}
}
func (S *Scanner) scanComment() {
// initial '/' already consumed; S.ch == '/' || S.ch == '*'
offs := S.offset - 1 // position of initial '/'
- col := S.column - 1
- pos := token.Position{S.filename, S.offset - 1, S.line, S.column - 1}
if S.ch == '/' {
//-style comment
for S.ch != '\n' && S.ch >= 0 {
S.next()
}
- if col == 1 {
+ if offs == S.lineOffset {
// comment starts at the beginning of the current line
S.interpretLineComment(S.src[offs:S.offset])
}
}
}
- S.errorAt(pos, "comment not terminated")
+ S.error(offs, "comment not terminated")
}
func (S *Scanner) findLineEnd() bool {
// initial '/' already consumed
- defer func(line, col, offs int) {
+ defer func(offs int) {
// reset scanner state to where it was upon calling findLineEnd
- // (we don't scan //line comments and ignore errors thus
- // S.filename and S.ErrorCount don't change)
- S.line = line
- S.column = col
S.ch = '/'
S.offset = offs
S.rdOffset = offs + 1
S.next() // consume initial '/' again
- }(S.line, S.column-1, S.offset-1)
+ }(S.offset - 1)
// read ahead until a newline, EOF, or non-comment token is found
for S.ch == '/' || S.ch == '*' {
if S.ch == '0' {
// int or float
- pos := token.Position{S.filename, S.offset, S.line, S.column}
+ offs := S.offset
S.next()
if S.ch == 'x' || S.ch == 'X' {
// hexadecimal int
}
// octal int
if seenDecimalDigit {
- S.errorAt(pos, "illegal octal number")
+ S.error(offs, "illegal octal number")
}
}
goto exit
func (S *Scanner) scanEscape(quote int) {
- pos := token.Position{S.filename, S.offset, S.line, S.column}
+ offs := S.offset
var i, base, max uint32
switch S.ch {
i, base, max = 8, 16, unicode.MaxRune
default:
S.next() // always make progress
- S.errorAt(pos, "unknown escape sequence")
+ S.error(offs, "unknown escape sequence")
return
}
for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
d := uint32(digitVal(S.ch))
if d >= base {
- S.error("illegal character in escape sequence")
+ S.error(S.offset, "illegal character in escape sequence")
break
}
x = x*base + d
S.next()
}
if x > max || 0xd800 <= x && x < 0xe000 {
- S.errorAt(pos, "escape sequence is invalid Unicode code point")
+ S.error(offs, "escape sequence is invalid Unicode code point")
}
}
func (S *Scanner) scanChar() {
// '\'' opening already consumed
- pos := token.Position{S.filename, S.offset - 1, S.line, S.column - 1}
+ offs := S.offset - 1
n := 0
for S.ch != '\'' {
n++
S.next()
if ch == '\n' || ch < 0 {
- S.errorAt(pos, "character literal not terminated")
+ S.error(offs, "character literal not terminated")
n = 1
break
}
S.next()
if n != 1 {
- S.errorAt(pos, "illegal character literal")
+ S.error(offs, "illegal character literal")
}
}
func (S *Scanner) scanString() {
// '"' opening already consumed
- pos := token.Position{S.filename, S.offset - 1, S.line, S.column - 1}
+ offs := S.offset - 1
for S.ch != '"' {
ch := S.ch
S.next()
if ch == '\n' || ch < 0 {
- S.errorAt(pos, "string not terminated")
+ S.error(offs, "string not terminated")
break
}
if ch == '\\' {
func (S *Scanner) scanRawString() {
// '`' opening already consumed
- pos := token.Position{S.filename, S.offset - 1, S.line, S.column - 1}
+ offs := S.offset - 1
for S.ch != '`' {
ch := S.ch
S.next()
if ch < 0 {
- S.errorAt(pos, "string not terminated")
+ S.error(offs, "string not terminated")
break
}
}
// must check the scanner's ErrorCount or the number of calls
// of the error handler, if there was one installed.
//
-func (S *Scanner) Scan() (pos token.Position, tok token.Token, lit []byte) {
+// Scan adds line information to the file added to the file
+// set with Init. Token positions are relative to that file
+// and thus relative to the file set.
+//
+func (S *Scanner) Scan() (token.Pos, token.Token, []byte) {
scanAgain:
S.skipWhitespace()
// current token start
insertSemi := false
- pos, tok = token.Position{S.filename, S.offset, S.line, S.column}, token.ILLEGAL
offs := S.offset
+ tok := token.ILLEGAL
// determine token value
switch ch := S.ch; {
case -1:
if S.insertSemi {
S.insertSemi = false // EOF consumed
- return pos, token.SEMICOLON, newline
+ return S.file.Pos(offs), token.SEMICOLON, newline
}
tok = token.EOF
case '\n':
// set in the first place and exited early
// from S.skipWhitespace()
S.insertSemi = false // newline consumed
- return pos, token.SEMICOLON, newline
+ return S.file.Pos(offs), token.SEMICOLON, newline
case '"':
insertSemi = true
tok = token.STRING
case '/':
if S.ch == '/' || S.ch == '*' {
// comment
- line := S.line
- col := S.column - 1 // beginning of comment
if S.insertSemi && S.findLineEnd() {
// reset position to the beginning of the comment
- S.line = line
- S.column = col
S.ch = '/'
S.offset = offs
S.rdOffset = offs + 1
S.insertSemi = false // newline consumed
- return pos, token.SEMICOLON, newline
+ return S.file.Pos(offs), token.SEMICOLON, newline
}
S.scanComment()
if S.mode&ScanComments == 0 {
tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
default:
if S.mode&AllowIllegalChars == 0 {
- S.errorAt(pos, "illegal character "+charString(ch))
+ S.error(offs, "illegal character "+charString(ch))
}
insertSemi = S.insertSemi // preserve insertSemi info
}
if S.mode&InsertSemis != 0 {
S.insertSemi = insertSemi
}
- return pos, tok, S.src[offs:S.offset]
+ return S.file.Pos(offs), tok, S.src[offs:S.offset]
}
// false (usually when the token value is token.EOF). The result is the number
// of errors encountered.
//
-func Tokenize(filename string, src []byte, err ErrorHandler, mode uint, f func(pos token.Position, tok token.Token, lit []byte) bool) int {
+func Tokenize(set *token.FileSet, filename string, src []byte, err ErrorHandler, mode uint, f func(pos token.Pos, tok token.Token, lit []byte) bool) int {
var s Scanner
- s.Init(filename, src, err, mode)
+ s.Init(set, filename, src, err, mode)
for f(s.Scan()) {
// action happens in f
}
)
+var fset = token.NewFileSet()
+
+
const /* class */ (
special = iota
literal
}
-func checkPos(t *testing.T, lit string, pos, expected token.Position) {
+func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
+ pos := fset.Position(p)
if pos.Filename != expected.Filename {
t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename)
}
for _, e := range tokens {
src += e.lit + whitespace
}
- src_linecount := newlineCount(src)
+ src_linecount := newlineCount(src) + 1
whitespace_linecount := newlineCount(whitespace)
// verify scan
index := 0
epos := token.Position{"", 0, 1, 1} // expected position
- nerrors := Tokenize("", []byte(src), &testErrorHandler{t}, ScanComments,
- func(pos token.Position, tok token.Token, litb []byte) bool {
+ nerrors := Tokenize(fset, "", []byte(src), &testErrorHandler{t}, ScanComments,
+ func(pos token.Pos, tok token.Token, litb []byte) bool {
e := elt{token.EOF, "", special}
if index < len(tokens) {
e = tokens[index]
func checkSemi(t *testing.T, line string, mode uint) {
var S Scanner
- S.Init("TestSemis", []byte(line), nil, mode)
+ file := S.Init(fset, "TestSemis", []byte(line), nil, mode)
pos, tok, lit := S.Scan()
for tok != token.EOF {
if tok == token.ILLEGAL {
semiLit = ";"
}
// next token must be a semicolon
- semiPos := pos
+ semiPos := file.Position(pos)
semiPos.Offset++
semiPos.Column++
pos, tok, lit = S.Scan()
// verify scan
var S Scanner
- S.Init("TestLineComments", []byte(src), nil, 0)
+ file := S.Init(fset, "TestLineComments", []byte(src), nil, 0)
for _, s := range segments {
- pos, _, lit := S.Scan()
- checkPos(t, string(lit), pos, token.Position{s.filename, pos.Offset, s.line, pos.Column})
+ p, _, lit := S.Scan()
+ pos := file.Position(p)
+ checkPos(t, string(lit), p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
}
if S.ErrorCount != 0 {
var s Scanner
// 1st init
- s.Init("", []byte("if true { }"), nil, 0)
+ src1 := "if true { }"
+ f1 := s.Init(fset, "", []byte(src1), nil, 0)
+ if f1.Size() != len(src1) {
+ t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
+ }
s.Scan() // if
s.Scan() // true
_, tok, _ := s.Scan() // {
}
// 2nd init
- s.Init("", []byte("go true { ]"), nil, 0)
+ src2 := "go true { ]"
+ f2 := s.Init(fset, "", []byte(src2), nil, 0)
+ if f2.Size() != len(src2) {
+ t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
+ }
_, tok, _ = s.Scan() // go
if tok != token.GO {
t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO)
var s Scanner
const src = "*?*$*@*"
- s.Init("", []byte(src), &testErrorHandler{t}, AllowIllegalChars)
+ file := s.Init(fset, "", []byte(src), &testErrorHandler{t}, AllowIllegalChars)
for offs, ch := range src {
pos, tok, lit := s.Scan()
- if pos.Offset != offs {
- t.Errorf("bad position for %s: got %d, expected %d", string(lit), pos.Offset, offs)
+ if poffs := file.Offset(pos); poffs != offs {
+ t.Errorf("bad position for %s: got %d, expected %d", string(lit), poffs, offs)
}
if tok == token.ILLEGAL && string(lit) != string(ch) {
t.Errorf("bad token: got %s, expected %s", string(lit), string(ch))
"@ @ @" // original file, line 1 again
v := new(ErrorVector)
- nerrors := Tokenize("File1", []byte(src), v, 0,
- func(pos token.Position, tok token.Token, litb []byte) bool {
+ nerrors := Tokenize(fset, "File1", []byte(src), v, 0,
+ func(pos token.Pos, tok token.Token, litb []byte) bool {
return tok != token.EOF
})
func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
var s Scanner
var h errorCollector
- s.Init("", []byte(src), &h, ScanComments)
+ s.Init(fset, "", []byte(src), &h, ScanComments)
_, tok0, _ := s.Scan()
_, tok1, _ := s.Scan()
if tok0 != tok {
// Position converts a Pos in the fileset into a general Position.
func (s *FileSet) Position(p Pos) (pos Position) {
if p != NoPos {
+ // TODO(gri) consider optimizing the case where p
+ // is in the last file addded, or perhaps
+ // looked at - will eliminate one level
+ // of search
s.mutex.RLock()
if f := s.file(p); f != nil {
offset := int(p) - f.base
// Offset returns the offset for the given file position p;
-// p must be a Pos value in that file.
+// p must be a valid Pos value in that file.
// f.Offset(f.Pos(offset)) == offset.
//
func (f *File) Offset(p Pos) int {
}
-// Position returns the Position value for the given file offset;
-// the offset must be <= f.Size().
+// Line returns the line number for the given file position p;
+// p must be a Pos value in that file or NoPos.
//
-func (f *File) Position(offset int) Position {
- if offset > f.size {
- panic("illegal file offset")
+func (f *File) Line(p Pos) int {
+ // TODO(gri) this can be implemented much more efficiently
+ return f.Position(p).Line
+}
+
+
+// Position returns the Position value for the given file position p;
+// p must be a Pos value in that file or NoPos.
+//
+func (f *File) Position(p Pos) (pos Position) {
+ if p != NoPos {
+ if int(p) < f.base || int(p) > f.base+f.size {
+ panic("illegal Pos value")
+ }
+ // TODO(gri) compute Position directly instead of going via the fset!
+ pos = f.set.Position(p)
}
- return f.set.Position(Pos(offset + f.base))
+ return
}
}
line, col := linecol(lines, offs)
msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
- checkPos(t, msg, f.Position(offs), Position{f.Name(), offs, line, col})
+ checkPos(t, msg, f.Position(f.Pos(offs)), Position{f.Name(), offs, line, col})
checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col})
}
}
p := f.Pos(offs)
_, col := linecol(lines, offs)
msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
- checkPos(t, msg, f.Position(offs), Position{"bar", offs, 42, col})
+ checkPos(t, msg, f.Position(f.Pos(offs)), Position{"bar", offs, 42, col})
checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col})
}
}
// objPos computes the source position of the declaration of an object name.
// Only required for error reporting, so doesn't have to be fast.
-func objPos(obj *ast.Object) (pos token.Position) {
+func objPos(obj *ast.Object) (pos token.Pos) {
switch d := obj.Decl.(type) {
case *ast.Field:
for _, n := range d.Names {
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+// INCOMPLETE PACKAGE.
// This package implements typechecking of a Go AST.
// The result of the typecheck is an augmented AST
// with object and type information for each identifier.
// If errors are reported, the AST may be incompletely augmented (fields
// may be nil) or contain incomplete object, type, or scope information.
//
-func CheckPackage(pkg *ast.Package, importer Importer) os.Error {
+func CheckPackage(fset *token.FileSet, pkg *ast.Package, importer Importer) os.Error {
var tc typechecker
+ tc.fset = fset
tc.importer = importer
tc.checkPackage(pkg)
return tc.GetError(scanner.Sorted)
// CheckPackage. If the complete package consists of more than just
// one file, the file may not typecheck without errors.
//
-func CheckFile(file *ast.File, importer Importer) os.Error {
+func CheckFile(fset *token.FileSet, file *ast.File, importer Importer) os.Error {
// create a single-file dummy package
- pkg := &ast.Package{file.Name.Name, nil, map[string]*ast.File{file.Name.NamePos.Filename: file}}
- return CheckPackage(pkg, importer)
+ pkg := &ast.Package{file.Name.Name, nil, map[string]*ast.File{fset.Position(file.Name.NamePos).Filename: file}}
+ return CheckPackage(fset, pkg, importer)
}
// Typechecker state
type typechecker struct {
+ fset *token.FileSet
scanner.ErrorVector
importer Importer
topScope *ast.Scope // current top-most scope
}
-func (tc *typechecker) Errorf(pos token.Position, format string, args ...interface{}) {
- tc.Error(pos, fmt.Sprintf(format, args...))
+func (tc *typechecker) Errorf(pos token.Pos, format string, args ...interface{}) {
+ tc.Error(tc.fset.Position(pos), fmt.Sprintf(format, args...))
}
const testDir = "./testdata" // location of test packages
+var fset = token.NewFileSet()
+
var (
pkgPat = flag.String("pkg", ".*", "regular expression to select test packages by package name")
trace = flag.Bool("trace", false, "print package names")
}
var s scanner.Scanner
- s.Init(filename, src, nil, scanner.ScanComments)
- var prev token.Position // position of last non-comment token
+ s.Init(fset, filename, src, nil, scanner.ScanComments)
+ var prev token.Pos // position of last non-comment token
loop:
for {
pos, tok, lit := s.Scan()
case token.COMMENT:
s := errRx.FindSubmatch(lit)
if len(s) == 2 {
- list = append(list, &scanner.Error{prev, string(s[1])})
+ list = append(list, &scanner.Error{fset.Position(prev), string(s[1])})
}
default:
prev = pos
t.Fatalf("illegal flag value %q: %s", *pkgPat, err)
}
- pkgs, err := parser.ParseDir(testDir, testFilter, 0)
+ pkgs, err := parser.ParseDir(fset, testDir, testFilter, 0)
if err != nil {
scanner.PrintError(os.Stderr, err)
t.Fatalf("packages in %s contain syntax errors", testDir)
}
xlist := expectedErrors(t, pkg)
- err := CheckPackage(pkg, nil)
+ err := CheckPackage(fset, pkg, nil)
if err != nil {
if elist, ok := err.(scanner.ErrorList); ok {
// verify that errors match
func g(list []ast.Expr) {
n := len(list)-1;
- println(list[n].Pos().Line);
+ println(list[n].Pos());
}
// f is the same as g except that the expression assigned to n is inlined.
func f(list []ast.Expr) {
// n := len(list)-1;
- println(list[len(list)-1 /* n */].Pos().Line);
+ println(list[len(list)-1 /* n */].Pos());
}