]> Cypherpunks repositories - gostls13.git/commitdiff
go/doc: first steps towards cleaning up go/doc
authorRobert Griesemer <gri@golang.org>
Tue, 10 Jan 2012 00:14:01 +0000 (16:14 -0800)
committerRobert Griesemer <gri@golang.org>
Tue, 10 Jan 2012 00:14:01 +0000 (16:14 -0800)
- separated exported data structures from doc reader
  by extracting all exported data structures into doc.go
  and moving the implementation into reader.go
- added missing documentation comments
- no API or semantic changes (but moved positions of
  PackageDoc.Doc and TypeDoc.Decl field up for consistency)
- runs all tests

R=golang-dev, r
CC=golang-dev
https://golang.org/cl/5527063

17 files changed:
src/buildscript_darwin_386.sh
src/buildscript_darwin_amd64.sh
src/buildscript_freebsd_386.sh
src/buildscript_freebsd_amd64.sh
src/buildscript_linux_386.sh
src/buildscript_linux_amd64.sh
src/buildscript_linux_arm.sh
src/buildscript_netbsd_386.sh
src/buildscript_netbsd_amd64.sh
src/buildscript_openbsd_386.sh
src/buildscript_openbsd_amd64.sh
src/buildscript_plan9_386.sh
src/buildscript_windows_386.sh
src/buildscript_windows_amd64.sh
src/pkg/go/doc/Makefile
src/pkg/go/doc/doc.go
src/pkg/go/doc/reader.go [new file with mode: 0644]

index 44c33cef1faa28b745ccc0086c07d2c8ba951693..bb292d1f3ac55162cbca22075cf878a57ee17642 100755 (executable)
@@ -460,7 +460,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/darwin_386/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.8
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/darwin_386/go/doc.a
 
index fcc697552cac61bcdd6deb021ddcf4d4d0e98128..c537848a53f204c32213446a379a1d2896e861d6 100755 (executable)
@@ -459,7 +459,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/darwin_amd64/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.6
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/darwin_amd64/go/doc.a
 
index a444e94771c9740ae149e2e175beeee2e5714381..4f5cfe6daecd41ac505d7a2a81101cc889a57619 100755 (executable)
@@ -460,7 +460,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/freebsd_386/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.8
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/freebsd_386/go/doc.a
 
index d5dca47505ea731606326ebe449b3f4dda8bb578..d296eb37f521dbb89541d75a2890a17bf1111b31 100755 (executable)
@@ -459,7 +459,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/freebsd_amd64/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.6
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/freebsd_amd64/go/doc.a
 
index 09260ae9b3c82d61cd2de35ab5289e6ac4764f3b..7d26ba57741e79269329525b9fc1a5d2844efa17 100755 (executable)
@@ -460,7 +460,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/linux_386/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.8
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/linux_386/go/doc.a
 
index cc1ab834ea483c623b7e257563a68294f03328a8..87ce176f2ff2dbd98a0c4678eb38f8f1becab1f7 100755 (executable)
@@ -459,7 +459,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/linux_amd64/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.6
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/linux_amd64/go/doc.a
 
index 9e18fe6fddde55972c60cb9fe3f598f82ec79f7d..d063ba0f2760f682d9c491b7d954abcbeb6ca942 100755 (executable)
@@ -463,7 +463,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/linux_arm/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-5g -o "$WORK"/go/doc/_obj/_go_.5 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+5g -o "$WORK"/go/doc/_obj/_go_.5 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.5
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/linux_arm/go/doc.a
 
index 4112d905e62822a84d10ea22b6a5490c886fe282..fb5836b18e55ea03e29122b21ea8d727b199de5a 100755 (executable)
@@ -460,7 +460,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/netbsd_386/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.8
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/netbsd_386/go/doc.a
 
index c83a2dadf152061743aed4a37016f845404bdc06..c75062d60489bea48a9c55390f3bb862210aa4bf 100755 (executable)
@@ -459,7 +459,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/netbsd_amd64/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.6
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/netbsd_amd64/go/doc.a
 
index e63b97aad6bf01d7a11afda60a92bab254a7c5fa..0435a2fdbc2285a365d59a576955cc889ccb78a4 100755 (executable)
@@ -460,7 +460,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/openbsd_386/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.8
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/openbsd_386/go/doc.a
 
index 8099e999a7f5ada7defd7141fc565119fa7ad9ea..89907c0a93f4023859145751253cb9be9fdfcc6d 100755 (executable)
@@ -459,7 +459,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/openbsd_amd64/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.6
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/openbsd_amd64/go/doc.a
 
index 07cb12a3ce6e132b2595eeb7542dbfdaa76453c7..304c12e58a2125a40b54cf13f5c8787002015c51 100755 (executable)
@@ -460,7 +460,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/plan9_386/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.8
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/plan9_386/go/doc.a
 
index 8518fd19e27498afd7233836130277af87559684..0aeacfba0f0eaedad44d79fc4c09c82783e242b7 100755 (executable)
@@ -462,7 +462,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/windows_386/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+8g -o "$WORK"/go/doc/_obj/_go_.8 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.8
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/windows_386/go/doc.a
 
index d27df78664708f48f6ca6e4b579c10e7cf2d012d..f2d3557c20c1302c20fda31de7dfef056b97f84f 100755 (executable)
@@ -461,7 +461,7 @@ cp "$WORK"/text/template.a "$GOROOT"/pkg/windows_amd64/text/template.a
 
 mkdir -p "$WORK"/go/doc/_obj/
 cd "$GOROOT"/src/pkg/go/doc
-6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go
+6g -o "$WORK"/go/doc/_obj/_go_.6 -p go/doc -I "$WORK" ./comment.go ./doc.go ./example.go ./exports.go ./filter.go ./reader.go
 gopack grc "$WORK"/go/doc.a "$WORK"/go/doc/_obj/_go_.6
 cp "$WORK"/go/doc.a "$GOROOT"/pkg/windows_amd64/go/doc.a
 
index b27b85abea9e503d9d748fea58ddf1890127980f..ac2eeb96272d98334353ba0eded4799e55c73cee 100644 (file)
@@ -11,6 +11,7 @@ GOFILES=\
        example.go\
        exports.go\
        filter.go\
+       reader.go\
 
 include ../../../Make.pkg
 
index 1bb22416c78ad9f81a5103a50cf145c83cc94a99..044e996a9e4bbfa46e511ab96d5dcfcf342313fc 100644 (file)
 // Package doc extracts source code documentation from a Go AST.
 package doc
 
-import (
-       "go/ast"
-       "go/token"
-       "regexp"
-       "sort"
-)
+import "go/ast"
 
-// ----------------------------------------------------------------------------
-// Collection of documentation info
-
-// embeddedType describes the type of an anonymous field.
-//
-type embeddedType struct {
-       typ *typeInfo // the corresponding base type
-       ptr bool      // if set, the anonymous field type is a pointer
-}
-
-type typeInfo struct {
-       // len(decl.Specs) == 1, and the element type is *ast.TypeSpec
-       // if the type declaration hasn't been seen yet, decl is nil
-       decl     *ast.GenDecl
-       embedded []embeddedType
-       forward  *TypeDoc // forward link to processed type documentation
-
-       // declarations associated with the type
-       values    []*ast.GenDecl // consts and vars
-       factories map[string]*ast.FuncDecl
-       methods   map[string]*ast.FuncDecl
-}
-
-func (info *typeInfo) addEmbeddedType(embedded *typeInfo, isPtr bool) {
-       info.embedded = append(info.embedded, embeddedType{embedded, isPtr})
-}
-
-// docReader accumulates documentation for a single package.
-// It modifies the AST: Comments (declaration documentation)
-// that have been collected by the DocReader are set to nil
-// in the respective AST nodes so that they are not printed
-// twice (once when printing the documentation and once when
-// printing the corresponding AST node).
-//
-type docReader struct {
-       doc      *ast.CommentGroup // package documentation, if any
-       pkgName  string
-       values   []*ast.GenDecl // consts and vars
-       types    map[string]*typeInfo
-       embedded map[string]*typeInfo // embedded types, possibly not exported
-       funcs    map[string]*ast.FuncDecl
-       bugs     []*ast.CommentGroup
-}
-
-func (doc *docReader) init(pkgName string) {
-       doc.pkgName = pkgName
-       doc.types = make(map[string]*typeInfo)
-       doc.embedded = make(map[string]*typeInfo)
-       doc.funcs = make(map[string]*ast.FuncDecl)
-}
-
-func (doc *docReader) addDoc(comments *ast.CommentGroup) {
-       if doc.doc == nil {
-               // common case: just one package comment
-               doc.doc = comments
-               return
-       }
-       // More than one package comment: Usually there will be only
-       // one file with a package comment, but it's better to collect
-       // all comments than drop them on the floor.
-       blankComment := &ast.Comment{token.NoPos, "//"}
-       list := append(doc.doc.List, blankComment)
-       doc.doc.List = append(list, comments.List...)
-}
-
-func (doc *docReader) lookupTypeInfo(name string) *typeInfo {
-       if name == "" || name == "_" {
-               return nil // no type docs for anonymous types
-       }
-       if info, found := doc.types[name]; found {
-               return info
-       }
-       // type wasn't found - add one without declaration
-       info := &typeInfo{
-               factories: make(map[string]*ast.FuncDecl),
-               methods:   make(map[string]*ast.FuncDecl),
-       }
-       doc.types[name] = info
-       return info
-}
-
-func baseTypeName(typ ast.Expr, allTypes bool) string {
-       switch t := typ.(type) {
-       case *ast.Ident:
-               // if the type is not exported, the effect to
-               // a client is as if there were no type name
-               if t.IsExported() || allTypes {
-                       return t.Name
-               }
-       case *ast.StarExpr:
-               return baseTypeName(t.X, allTypes)
-       }
-       return ""
-}
-
-func (doc *docReader) addValue(decl *ast.GenDecl) {
-       // determine if decl should be associated with a type
-       // Heuristic: For each typed entry, determine the type name, if any.
-       //            If there is exactly one type name that is sufficiently
-       //            frequent, associate the decl with the respective type.
-       domName := ""
-       domFreq := 0
-       prev := ""
-       for _, s := range decl.Specs {
-               if v, ok := s.(*ast.ValueSpec); ok {
-                       name := ""
-                       switch {
-                       case v.Type != nil:
-                               // a type is present; determine its name
-                               name = baseTypeName(v.Type, false)
-                       case decl.Tok == token.CONST:
-                               // no type is present but we have a constant declaration;
-                               // use the previous type name (w/o more type information
-                               // we cannot handle the case of unnamed variables with
-                               // initializer expressions except for some trivial cases)
-                               name = prev
-                       }
-                       if name != "" {
-                               // entry has a named type
-                               if domName != "" && domName != name {
-                                       // more than one type name - do not associate
-                                       // with any type
-                                       domName = ""
-                                       break
-                               }
-                               domName = name
-                               domFreq++
-                       }
-                       prev = name
-               }
-       }
-
-       // determine values list
-       const threshold = 0.75
-       values := &doc.values
-       if domName != "" && domFreq >= int(float64(len(decl.Specs))*threshold) {
-               // typed entries are sufficiently frequent
-               typ := doc.lookupTypeInfo(domName)
-               if typ != nil {
-                       values = &typ.values // associate with that type
-               }
-       }
-
-       *values = append(*values, decl)
-}
-
-// Helper function to set the table entry for function f. Makes sure that
-// at least one f with associated documentation is stored in table, if there
-// are multiple f's with the same name.
-func setFunc(table map[string]*ast.FuncDecl, f *ast.FuncDecl) {
-       name := f.Name.Name
-       if g, exists := table[name]; exists && g.Doc != nil {
-               // a function with the same name has already been registered;
-               // since it has documentation, assume f is simply another
-               // implementation and ignore it
-               // TODO(gri) consider collecting all functions, or at least
-               //           all comments
-               return
-       }
-       // function doesn't exist or has no documentation; use f
-       table[name] = f
+// PackageDoc is the documentation for an entire package.
+type PackageDoc struct {
+       Doc         string
+       PackageName string
+       ImportPath  string
+       Filenames   []string
+       Consts      []*ValueDoc
+       Types       []*TypeDoc
+       Vars        []*ValueDoc
+       Funcs       []*FuncDoc
+       Bugs        []string
 }
 
-func (doc *docReader) addFunc(fun *ast.FuncDecl) {
-       // strip function body
-       fun.Body = nil
-
-       // determine if it should be associated with a type
-       if fun.Recv != nil {
-               // method
-               typ := doc.lookupTypeInfo(baseTypeName(fun.Recv.List[0].Type, false))
-               if typ != nil {
-                       // exported receiver type
-                       setFunc(typ.methods, fun)
-               }
-               // otherwise don't show the method
-               // TODO(gri): There may be exported methods of non-exported types
-               // that can be called because of exported values (consts, vars, or
-               // function results) of that type. Could determine if that is the
-               // case and then show those methods in an appropriate section.
-               return
-       }
-
-       // perhaps a factory function
-       // determine result type, if any
-       if fun.Type.Results.NumFields() >= 1 {
-               res := fun.Type.Results.List[0]
-               if len(res.Names) <= 1 {
-                       // exactly one (named or anonymous) result associated
-                       // with the first type in result signature (there may
-                       // be more than one result)
-                       tname := baseTypeName(res.Type, false)
-                       typ := doc.lookupTypeInfo(tname)
-                       if typ != nil {
-                               // named and exported result type
-                               setFunc(typ.factories, fun)
-                               return
-                       }
-               }
-       }
+// Value is the documentation for a (possibly grouped) var or const declaration.
+type ValueDoc struct {
+       Doc  string
+       Decl *ast.GenDecl
 
-       // ordinary function
-       setFunc(doc.funcs, fun)
+       order int
 }
 
-func (doc *docReader) addDecl(decl ast.Decl) {
-       switch d := decl.(type) {
-       case *ast.GenDecl:
-               if len(d.Specs) > 0 {
-                       switch d.Tok {
-                       case token.CONST, token.VAR:
-                               // constants and variables are always handled as a group
-                               doc.addValue(d)
-                       case token.TYPE:
-                               // types are handled individually
-                               for _, spec := range d.Specs {
-                                       tspec := spec.(*ast.TypeSpec)
-                                       // add the type to the documentation
-                                       info := doc.lookupTypeInfo(tspec.Name.Name)
-                                       if info == nil {
-                                               continue // no name - ignore the type
-                                       }
-                                       // Make a (fake) GenDecl node for this TypeSpec
-                                       // (we need to do this here - as opposed to just
-                                       // for printing - so we don't lose the GenDecl
-                                       // documentation). Since a new GenDecl node is
-                                       // created, there's no need to nil out d.Doc.
-                                       //
-                                       // TODO(gri): Consider just collecting the TypeSpec
-                                       // node (and copy in the GenDecl.doc if there is no
-                                       // doc in the TypeSpec - this is currently done in
-                                       // makeTypeDocs below). Simpler data structures, but
-                                       // would lose GenDecl documentation if the TypeSpec
-                                       // has documentation as well.
-                                       fake := &ast.GenDecl{d.Doc, d.Pos(), token.TYPE, token.NoPos,
-                                               []ast.Spec{tspec}, token.NoPos}
-                                       // A type should be added at most once, so info.decl
-                                       // should be nil - if it isn't, simply overwrite it.
-                                       info.decl = fake
-                                       // Look for anonymous fields that might contribute methods.
-                                       var fields *ast.FieldList
-                                       switch typ := spec.(*ast.TypeSpec).Type.(type) {
-                                       case *ast.StructType:
-                                               fields = typ.Fields
-                                       case *ast.InterfaceType:
-                                               fields = typ.Methods
-                                       }
-                                       if fields != nil {
-                                               for _, field := range fields.List {
-                                                       if len(field.Names) == 0 {
-                                                               // anonymous field - add corresponding type
-                                                               // to the info and collect it in doc
-                                                               name := baseTypeName(field.Type, true)
-                                                               if embedded := doc.lookupTypeInfo(name); embedded != nil {
-                                                                       _, ptr := field.Type.(*ast.StarExpr)
-                                                                       info.addEmbeddedType(embedded, ptr)
-                                                               }
-                                                       }
-                                               }
-                                       }
-                               }
-                       }
-               }
-       case *ast.FuncDecl:
-               doc.addFunc(d)
-       }
-}
+// TypeDoc is the documentation for type declaration.
+type TypeDoc struct {
+       Doc       string
+       Type      *ast.TypeSpec
+       Decl      *ast.GenDecl
+       Consts    []*ValueDoc // sorted list of constants of (mostly) this type
+       Vars      []*ValueDoc // sorted list of variables of (mostly) this type
+       Factories []*FuncDoc  // sorted list of functions returning this type
+       Methods   []*FuncDoc  // sorted list of methods (including embedded ones) of this type
 
-func copyCommentList(list []*ast.Comment) []*ast.Comment {
-       return append([]*ast.Comment(nil), list...)
+       methods  []*FuncDoc // top-level methods only
+       embedded methodSet  // embedded methods only
+       order    int
 }
 
-var (
-       bug_markers = regexp.MustCompile("^/[/*][ \t]*BUG\\(.*\\):[ \t]*") // BUG(uid):
-       bug_content = regexp.MustCompile("[^ \n\r\t]+")                    // at least one non-whitespace char
-)
-
-// addFile adds the AST for a source file to the docReader.
-// Adding the same AST multiple times is a no-op.
-//
-func (doc *docReader) addFile(src *ast.File) {
-       // add package documentation
-       if src.Doc != nil {
-               doc.addDoc(src.Doc)
-               src.Doc = nil // doc consumed - remove from ast.File node
-       }
-
-       // add all declarations
-       for _, decl := range src.Decls {
-               doc.addDecl(decl)
-       }
-
-       // collect BUG(...) comments
-       for _, c := range src.Comments {
-               text := c.List[0].Text
-               if m := bug_markers.FindStringIndex(text); m != nil {
-                       // found a BUG comment; maybe empty
-                       if btxt := text[m[1]:]; bug_content.MatchString(btxt) {
-                               // non-empty BUG comment; collect comment without BUG prefix
-                               list := copyCommentList(c.List)
-                               list[0].Text = text[m[1]:]
-                               doc.bugs = append(doc.bugs, &ast.CommentGroup{list})
-                       }
-               }
-       }
-       src.Comments = nil // consumed unassociated comments - remove from ast.File node
+// Func is the documentation for a func declaration.
+type FuncDoc struct {
+       Doc  string
+       Recv ast.Expr // TODO(rsc): Would like string here
+       Name string
+       Decl *ast.FuncDecl
 }
 
+// NewPackageDoc computes the package documentation for the given package
+// and import path. If exportsOnly is set, only exported objects are
+// included in the documentation.
 func NewPackageDoc(pkg *ast.Package, importpath string, exportsOnly bool) *PackageDoc {
        var r docReader
        r.init(pkg.Name)
@@ -335,345 +69,3 @@ func NewPackageDoc(pkg *ast.Package, importpath string, exportsOnly bool) *Packa
        }
        return r.newDoc(importpath, filenames)
 }
-
-// ----------------------------------------------------------------------------
-// Conversion to external representation
-
-// ValueDoc is the documentation for a group of declared
-// values, either vars or consts.
-//
-type ValueDoc struct {
-       Doc   string
-       Decl  *ast.GenDecl
-       order int
-}
-
-type sortValueDoc []*ValueDoc
-
-func (p sortValueDoc) Len() int      { return len(p) }
-func (p sortValueDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
-
-func declName(d *ast.GenDecl) string {
-       if len(d.Specs) != 1 {
-               return ""
-       }
-
-       switch v := d.Specs[0].(type) {
-       case *ast.ValueSpec:
-               return v.Names[0].Name
-       case *ast.TypeSpec:
-               return v.Name.Name
-       }
-
-       return ""
-}
-
-func (p sortValueDoc) Less(i, j int) bool {
-       // sort by name
-       // pull blocks (name = "") up to top
-       // in original order
-       if ni, nj := declName(p[i].Decl), declName(p[j].Decl); ni != nj {
-               return ni < nj
-       }
-       return p[i].order < p[j].order
-}
-
-func makeValueDocs(list []*ast.GenDecl, tok token.Token) []*ValueDoc {
-       d := make([]*ValueDoc, len(list)) // big enough in any case
-       n := 0
-       for i, decl := range list {
-               if decl.Tok == tok {
-                       d[n] = &ValueDoc{CommentText(decl.Doc), decl, i}
-                       n++
-                       decl.Doc = nil // doc consumed - removed from AST
-               }
-       }
-       d = d[0:n]
-       sort.Sort(sortValueDoc(d))
-       return d
-}
-
-// FuncDoc is the documentation for a func declaration,
-// either a top-level function or a method function.
-//
-type FuncDoc struct {
-       Doc  string
-       Recv ast.Expr // TODO(rsc): Would like string here
-       Name string
-       Decl *ast.FuncDecl
-}
-
-type sortFuncDoc []*FuncDoc
-
-func (p sortFuncDoc) Len() int           { return len(p) }
-func (p sortFuncDoc) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-func (p sortFuncDoc) Less(i, j int) bool { return p[i].Name < p[j].Name }
-
-func makeFuncDocs(m map[string]*ast.FuncDecl) []*FuncDoc {
-       d := make([]*FuncDoc, len(m))
-       i := 0
-       for _, f := range m {
-               doc := new(FuncDoc)
-               doc.Doc = CommentText(f.Doc)
-               f.Doc = nil // doc consumed - remove from ast.FuncDecl node
-               if f.Recv != nil {
-                       doc.Recv = f.Recv.List[0].Type
-               }
-               doc.Name = f.Name.Name
-               doc.Decl = f
-               d[i] = doc
-               i++
-       }
-       sort.Sort(sortFuncDoc(d))
-       return d
-}
-
-type methodSet map[string]*FuncDoc
-
-func (mset methodSet) add(m *FuncDoc) {
-       if mset[m.Name] == nil {
-               mset[m.Name] = m
-       }
-}
-
-func (mset methodSet) sortedList() []*FuncDoc {
-       list := make([]*FuncDoc, len(mset))
-       i := 0
-       for _, m := range mset {
-               list[i] = m
-               i++
-       }
-       sort.Sort(sortFuncDoc(list))
-       return list
-}
-
-// TypeDoc is the documentation for a declared type.
-// Consts and Vars are sorted lists of constants and variables of (mostly) that type.
-// Factories is a sorted list of factory functions that return that type.
-// Methods is a sorted list of method functions on that type.
-type TypeDoc struct {
-       Doc       string
-       Type      *ast.TypeSpec
-       Consts    []*ValueDoc
-       Vars      []*ValueDoc
-       Factories []*FuncDoc
-       methods   []*FuncDoc // top-level methods only
-       embedded  methodSet  // embedded methods only
-       Methods   []*FuncDoc // all methods including embedded ones
-       Decl      *ast.GenDecl
-       order     int
-}
-
-type sortTypeDoc []*TypeDoc
-
-func (p sortTypeDoc) Len() int      { return len(p) }
-func (p sortTypeDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
-func (p sortTypeDoc) Less(i, j int) bool {
-       // sort by name
-       // pull blocks (name = "") up to top
-       // in original order
-       if ni, nj := p[i].Type.Name.Name, p[j].Type.Name.Name; ni != nj {
-               return ni < nj
-       }
-       return p[i].order < p[j].order
-}
-
-// NOTE(rsc): This would appear not to be correct for type ( )
-// blocks, but the doc extractor above has split them into
-// individual declarations.
-func (doc *docReader) makeTypeDocs(m map[string]*typeInfo) []*TypeDoc {
-       // TODO(gri) Consider computing the embedded method information
-       //           before calling makeTypeDocs. Then this function can
-       //           be single-phased again. Also, it might simplify some
-       //           of the logic.
-       //
-       // phase 1: associate collected declarations with TypeDocs
-       list := make([]*TypeDoc, len(m))
-       i := 0
-       for _, old := range m {
-               // all typeInfos should have a declaration associated with
-               // them after processing an entire package - be conservative
-               // and check
-               if decl := old.decl; decl != nil {
-                       typespec := decl.Specs[0].(*ast.TypeSpec)
-                       t := new(TypeDoc)
-                       doc := typespec.Doc
-                       typespec.Doc = nil // doc consumed - remove from ast.TypeSpec node
-                       if doc == nil {
-                               // no doc associated with the spec, use the declaration doc, if any
-                               doc = decl.Doc
-                       }
-                       decl.Doc = nil // doc consumed - remove from ast.Decl node
-                       t.Doc = CommentText(doc)
-                       t.Type = typespec
-                       t.Consts = makeValueDocs(old.values, token.CONST)
-                       t.Vars = makeValueDocs(old.values, token.VAR)
-                       t.Factories = makeFuncDocs(old.factories)
-                       t.methods = makeFuncDocs(old.methods)
-                       // The list of embedded types' methods is computed from the list
-                       // of embedded types, some of which may not have been processed
-                       // yet (i.e., their forward link is nil) - do this in a 2nd phase.
-                       // The final list of methods can only be computed after that -
-                       // do this in a 3rd phase.
-                       t.Decl = old.decl
-                       t.order = i
-                       old.forward = t // old has been processed
-                       list[i] = t
-                       i++
-               } else {
-                       // no corresponding type declaration found - move any associated
-                       // values, factory functions, and methods back to the top-level
-                       // so that they are not lost (this should only happen if a package
-                       // file containing the explicit type declaration is missing or if
-                       // an unqualified type name was used after a "." import)
-                       // 1) move values
-                       doc.values = append(doc.values, old.values...)
-                       // 2) move factory functions
-                       for name, f := range old.factories {
-                               doc.funcs[name] = f
-                       }
-                       // 3) move methods
-                       for name, f := range old.methods {
-                               // don't overwrite functions with the same name
-                               if _, found := doc.funcs[name]; !found {
-                                       doc.funcs[name] = f
-                               }
-                       }
-               }
-       }
-       list = list[0:i] // some types may have been ignored
-
-       // phase 2: collect embedded methods for each processed typeInfo
-       for _, old := range m {
-               if t := old.forward; t != nil {
-                       // old has been processed into t; collect embedded
-                       // methods for t from the list of processed embedded
-                       // types in old (and thus for which the methods are known)
-                       typ := t.Type
-                       if _, ok := typ.Type.(*ast.StructType); ok {
-                               // struct
-                               t.embedded = make(methodSet)
-                               collectEmbeddedMethods(t.embedded, old, typ.Name.Name)
-                       } else {
-                               // interface
-                               // TODO(gri) fix this
-                       }
-               }
-       }
-
-       // phase 3: compute final method set for each TypeDoc
-       for _, d := range list {
-               if len(d.embedded) > 0 {
-                       // there are embedded methods - exclude
-                       // the ones with names conflicting with
-                       // non-embedded methods
-                       mset := make(methodSet)
-                       // top-level methods have priority
-                       for _, m := range d.methods {
-                               mset.add(m)
-                       }
-                       // add non-conflicting embedded methods
-                       for _, m := range d.embedded {
-                               mset.add(m)
-                       }
-                       d.Methods = mset.sortedList()
-               } else {
-                       // no embedded methods
-                       d.Methods = d.methods
-               }
-       }
-
-       sort.Sort(sortTypeDoc(list))
-       return list
-}
-
-// collectEmbeddedMethods collects the embedded methods from all
-// processed embedded types found in info in mset. It considers
-// embedded types at the most shallow level first so that more
-// deeply nested embedded methods with conflicting names are
-// excluded.
-//
-func collectEmbeddedMethods(mset methodSet, info *typeInfo, recvTypeName string) {
-       for _, e := range info.embedded {
-               if e.typ.forward != nil { // == e was processed
-                       for _, m := range e.typ.forward.methods {
-                               mset.add(customizeRecv(m, e.ptr, recvTypeName))
-                       }
-                       collectEmbeddedMethods(mset, e.typ, recvTypeName)
-               }
-       }
-}
-
-func customizeRecv(m *FuncDoc, embeddedIsPtr bool, recvTypeName string) *FuncDoc {
-       if m == nil || m.Decl == nil || m.Decl.Recv == nil || len(m.Decl.Recv.List) != 1 {
-               return m // shouldn't happen, but be safe
-       }
-
-       // copy existing receiver field and set new type
-       // TODO(gri) is receiver type computation correct?
-       //           what about deeply nested embeddings?
-       newField := *m.Decl.Recv.List[0]
-       _, origRecvIsPtr := newField.Type.(*ast.StarExpr)
-       var typ ast.Expr = ast.NewIdent(recvTypeName)
-       if embeddedIsPtr || origRecvIsPtr {
-               typ = &ast.StarExpr{token.NoPos, typ}
-       }
-       newField.Type = typ
-
-       // copy existing receiver field list and set new receiver field
-       newFieldList := *m.Decl.Recv
-       newFieldList.List = []*ast.Field{&newField}
-
-       // copy existing function declaration and set new receiver field list
-       newFuncDecl := *m.Decl
-       newFuncDecl.Recv = &newFieldList
-
-       // copy existing function documentation and set new declaration
-       newM := *m
-       newM.Decl = &newFuncDecl
-       newM.Recv = typ
-
-       return &newM
-}
-
-func makeBugDocs(list []*ast.CommentGroup) []string {
-       d := make([]string, len(list))
-       for i, g := range list {
-               d[i] = CommentText(g)
-       }
-       return d
-}
-
-// PackageDoc is the documentation for an entire package.
-//
-type PackageDoc struct {
-       PackageName string
-       ImportPath  string
-       Filenames   []string
-       Doc         string
-       Consts      []*ValueDoc
-       Types       []*TypeDoc
-       Vars        []*ValueDoc
-       Funcs       []*FuncDoc
-       Bugs        []string
-}
-
-// newDoc returns the accumulated documentation for the package.
-//
-func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc {
-       p := new(PackageDoc)
-       p.PackageName = doc.pkgName
-       p.ImportPath = importpath
-       sort.Strings(filenames)
-       p.Filenames = filenames
-       p.Doc = CommentText(doc.doc)
-       // makeTypeDocs may extend the list of doc.values and
-       // doc.funcs and thus must be called before any other
-       // function consuming those lists
-       p.Types = doc.makeTypeDocs(doc.types)
-       p.Consts = makeValueDocs(doc.values, token.CONST)
-       p.Vars = makeValueDocs(doc.values, token.VAR)
-       p.Funcs = makeFuncDocs(doc.funcs)
-       p.Bugs = makeBugDocs(doc.bugs)
-       return p
-}
diff --git a/src/pkg/go/doc/reader.go b/src/pkg/go/doc/reader.go
new file mode 100644 (file)
index 0000000..4e4ed58
--- /dev/null
@@ -0,0 +1,613 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package doc extracts source code documentation from a Go AST.
+package doc
+
+import (
+       "go/ast"
+       "go/token"
+       "regexp"
+       "sort"
+)
+
+// ----------------------------------------------------------------------------
+// Collection of documentation info
+
+// embeddedType describes the type of an anonymous field.
+//
+type embeddedType struct {
+       typ *typeInfo // the corresponding base type
+       ptr bool      // if set, the anonymous field type is a pointer
+}
+
+type typeInfo struct {
+       // len(decl.Specs) == 1, and the element type is *ast.TypeSpec
+       // if the type declaration hasn't been seen yet, decl is nil
+       decl     *ast.GenDecl
+       embedded []embeddedType
+       forward  *TypeDoc // forward link to processed type documentation
+
+       // declarations associated with the type
+       values    []*ast.GenDecl // consts and vars
+       factories map[string]*ast.FuncDecl
+       methods   map[string]*ast.FuncDecl
+}
+
+func (info *typeInfo) addEmbeddedType(embedded *typeInfo, isPtr bool) {
+       info.embedded = append(info.embedded, embeddedType{embedded, isPtr})
+}
+
+// docReader accumulates documentation for a single package.
+// It modifies the AST: Comments (declaration documentation)
+// that have been collected by the DocReader are set to nil
+// in the respective AST nodes so that they are not printed
+// twice (once when printing the documentation and once when
+// printing the corresponding AST node).
+//
+type docReader struct {
+       doc      *ast.CommentGroup // package documentation, if any
+       pkgName  string
+       values   []*ast.GenDecl // consts and vars
+       types    map[string]*typeInfo
+       embedded map[string]*typeInfo // embedded types, possibly not exported
+       funcs    map[string]*ast.FuncDecl
+       bugs     []*ast.CommentGroup
+}
+
+func (doc *docReader) init(pkgName string) {
+       doc.pkgName = pkgName
+       doc.types = make(map[string]*typeInfo)
+       doc.embedded = make(map[string]*typeInfo)
+       doc.funcs = make(map[string]*ast.FuncDecl)
+}
+
+func (doc *docReader) addDoc(comments *ast.CommentGroup) {
+       if doc.doc == nil {
+               // common case: just one package comment
+               doc.doc = comments
+               return
+       }
+       // More than one package comment: Usually there will be only
+       // one file with a package comment, but it's better to collect
+       // all comments than drop them on the floor.
+       blankComment := &ast.Comment{token.NoPos, "//"}
+       list := append(doc.doc.List, blankComment)
+       doc.doc.List = append(list, comments.List...)
+}
+
+func (doc *docReader) lookupTypeInfo(name string) *typeInfo {
+       if name == "" || name == "_" {
+               return nil // no type docs for anonymous types
+       }
+       if info, found := doc.types[name]; found {
+               return info
+       }
+       // type wasn't found - add one without declaration
+       info := &typeInfo{
+               factories: make(map[string]*ast.FuncDecl),
+               methods:   make(map[string]*ast.FuncDecl),
+       }
+       doc.types[name] = info
+       return info
+}
+
+func baseTypeName(typ ast.Expr, allTypes bool) string {
+       switch t := typ.(type) {
+       case *ast.Ident:
+               // if the type is not exported, the effect to
+               // a client is as if there were no type name
+               if t.IsExported() || allTypes {
+                       return t.Name
+               }
+       case *ast.StarExpr:
+               return baseTypeName(t.X, allTypes)
+       }
+       return ""
+}
+
+func (doc *docReader) addValue(decl *ast.GenDecl) {
+       // determine if decl should be associated with a type
+       // Heuristic: For each typed entry, determine the type name, if any.
+       //            If there is exactly one type name that is sufficiently
+       //            frequent, associate the decl with the respective type.
+       domName := ""
+       domFreq := 0
+       prev := ""
+       for _, s := range decl.Specs {
+               if v, ok := s.(*ast.ValueSpec); ok {
+                       name := ""
+                       switch {
+                       case v.Type != nil:
+                               // a type is present; determine its name
+                               name = baseTypeName(v.Type, false)
+                       case decl.Tok == token.CONST:
+                               // no type is present but we have a constant declaration;
+                               // use the previous type name (w/o more type information
+                               // we cannot handle the case of unnamed variables with
+                               // initializer expressions except for some trivial cases)
+                               name = prev
+                       }
+                       if name != "" {
+                               // entry has a named type
+                               if domName != "" && domName != name {
+                                       // more than one type name - do not associate
+                                       // with any type
+                                       domName = ""
+                                       break
+                               }
+                               domName = name
+                               domFreq++
+                       }
+                       prev = name
+               }
+       }
+
+       // determine values list
+       const threshold = 0.75
+       values := &doc.values
+       if domName != "" && domFreq >= int(float64(len(decl.Specs))*threshold) {
+               // typed entries are sufficiently frequent
+               typ := doc.lookupTypeInfo(domName)
+               if typ != nil {
+                       values = &typ.values // associate with that type
+               }
+       }
+
+       *values = append(*values, decl)
+}
+
+// Helper function to set the table entry for function f. Makes sure that
+// at least one f with associated documentation is stored in table, if there
+// are multiple f's with the same name.
+func setFunc(table map[string]*ast.FuncDecl, f *ast.FuncDecl) {
+       name := f.Name.Name
+       if g, exists := table[name]; exists && g.Doc != nil {
+               // a function with the same name has already been registered;
+               // since it has documentation, assume f is simply another
+               // implementation and ignore it
+               // TODO(gri) consider collecting all functions, or at least
+               //           all comments
+               return
+       }
+       // function doesn't exist or has no documentation; use f
+       table[name] = f
+}
+
+func (doc *docReader) addFunc(fun *ast.FuncDecl) {
+       // strip function body
+       fun.Body = nil
+
+       // determine if it should be associated with a type
+       if fun.Recv != nil {
+               // method
+               typ := doc.lookupTypeInfo(baseTypeName(fun.Recv.List[0].Type, false))
+               if typ != nil {
+                       // exported receiver type
+                       setFunc(typ.methods, fun)
+               }
+               // otherwise don't show the method
+               // TODO(gri): There may be exported methods of non-exported types
+               // that can be called because of exported values (consts, vars, or
+               // function results) of that type. Could determine if that is the
+               // case and then show those methods in an appropriate section.
+               return
+       }
+
+       // perhaps a factory function
+       // determine result type, if any
+       if fun.Type.Results.NumFields() >= 1 {
+               res := fun.Type.Results.List[0]
+               if len(res.Names) <= 1 {
+                       // exactly one (named or anonymous) result associated
+                       // with the first type in result signature (there may
+                       // be more than one result)
+                       tname := baseTypeName(res.Type, false)
+                       typ := doc.lookupTypeInfo(tname)
+                       if typ != nil {
+                               // named and exported result type
+                               setFunc(typ.factories, fun)
+                               return
+                       }
+               }
+       }
+
+       // ordinary function
+       setFunc(doc.funcs, fun)
+}
+
+func (doc *docReader) addDecl(decl ast.Decl) {
+       switch d := decl.(type) {
+       case *ast.GenDecl:
+               if len(d.Specs) > 0 {
+                       switch d.Tok {
+                       case token.CONST, token.VAR:
+                               // constants and variables are always handled as a group
+                               doc.addValue(d)
+                       case token.TYPE:
+                               // types are handled individually
+                               for _, spec := range d.Specs {
+                                       tspec := spec.(*ast.TypeSpec)
+                                       // add the type to the documentation
+                                       info := doc.lookupTypeInfo(tspec.Name.Name)
+                                       if info == nil {
+                                               continue // no name - ignore the type
+                                       }
+                                       // Make a (fake) GenDecl node for this TypeSpec
+                                       // (we need to do this here - as opposed to just
+                                       // for printing - so we don't lose the GenDecl
+                                       // documentation). Since a new GenDecl node is
+                                       // created, there's no need to nil out d.Doc.
+                                       //
+                                       // TODO(gri): Consider just collecting the TypeSpec
+                                       // node (and copy in the GenDecl.doc if there is no
+                                       // doc in the TypeSpec - this is currently done in
+                                       // makeTypeDocs below). Simpler data structures, but
+                                       // would lose GenDecl documentation if the TypeSpec
+                                       // has documentation as well.
+                                       fake := &ast.GenDecl{d.Doc, d.Pos(), token.TYPE, token.NoPos,
+                                               []ast.Spec{tspec}, token.NoPos}
+                                       // A type should be added at most once, so info.decl
+                                       // should be nil - if it isn't, simply overwrite it.
+                                       info.decl = fake
+                                       // Look for anonymous fields that might contribute methods.
+                                       var fields *ast.FieldList
+                                       switch typ := spec.(*ast.TypeSpec).Type.(type) {
+                                       case *ast.StructType:
+                                               fields = typ.Fields
+                                       case *ast.InterfaceType:
+                                               fields = typ.Methods
+                                       }
+                                       if fields != nil {
+                                               for _, field := range fields.List {
+                                                       if len(field.Names) == 0 {
+                                                               // anonymous field - add corresponding type
+                                                               // to the info and collect it in doc
+                                                               name := baseTypeName(field.Type, true)
+                                                               if embedded := doc.lookupTypeInfo(name); embedded != nil {
+                                                                       _, ptr := field.Type.(*ast.StarExpr)
+                                                                       info.addEmbeddedType(embedded, ptr)
+                                                               }
+                                                       }
+                                               }
+                                       }
+                               }
+                       }
+               }
+       case *ast.FuncDecl:
+               doc.addFunc(d)
+       }
+}
+
+func copyCommentList(list []*ast.Comment) []*ast.Comment {
+       return append([]*ast.Comment(nil), list...)
+}
+
+var (
+       bug_markers = regexp.MustCompile("^/[/*][ \t]*BUG\\(.*\\):[ \t]*") // BUG(uid):
+       bug_content = regexp.MustCompile("[^ \n\r\t]+")                    // at least one non-whitespace char
+)
+
+// addFile adds the AST for a source file to the docReader.
+// Adding the same AST multiple times is a no-op.
+//
+func (doc *docReader) addFile(src *ast.File) {
+       // add package documentation
+       if src.Doc != nil {
+               doc.addDoc(src.Doc)
+               src.Doc = nil // doc consumed - remove from ast.File node
+       }
+
+       // add all declarations
+       for _, decl := range src.Decls {
+               doc.addDecl(decl)
+       }
+
+       // collect BUG(...) comments
+       for _, c := range src.Comments {
+               text := c.List[0].Text
+               if m := bug_markers.FindStringIndex(text); m != nil {
+                       // found a BUG comment; maybe empty
+                       if btxt := text[m[1]:]; bug_content.MatchString(btxt) {
+                               // non-empty BUG comment; collect comment without BUG prefix
+                               list := copyCommentList(c.List)
+                               list[0].Text = text[m[1]:]
+                               doc.bugs = append(doc.bugs, &ast.CommentGroup{list})
+                       }
+               }
+       }
+       src.Comments = nil // consumed unassociated comments - remove from ast.File node
+}
+
+// ----------------------------------------------------------------------------
+// Conversion to external representation
+
+type sortValueDoc []*ValueDoc
+
+func (p sortValueDoc) Len() int      { return len(p) }
+func (p sortValueDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func declName(d *ast.GenDecl) string {
+       if len(d.Specs) != 1 {
+               return ""
+       }
+
+       switch v := d.Specs[0].(type) {
+       case *ast.ValueSpec:
+               return v.Names[0].Name
+       case *ast.TypeSpec:
+               return v.Name.Name
+       }
+
+       return ""
+}
+
+func (p sortValueDoc) Less(i, j int) bool {
+       // sort by name
+       // pull blocks (name = "") up to top
+       // in original order
+       if ni, nj := declName(p[i].Decl), declName(p[j].Decl); ni != nj {
+               return ni < nj
+       }
+       return p[i].order < p[j].order
+}
+
+func makeValueDocs(list []*ast.GenDecl, tok token.Token) []*ValueDoc {
+       d := make([]*ValueDoc, len(list)) // big enough in any case
+       n := 0
+       for i, decl := range list {
+               if decl.Tok == tok {
+                       d[n] = &ValueDoc{CommentText(decl.Doc), decl, i}
+                       n++
+                       decl.Doc = nil // doc consumed - removed from AST
+               }
+       }
+       d = d[0:n]
+       sort.Sort(sortValueDoc(d))
+       return d
+}
+
+type sortFuncDoc []*FuncDoc
+
+func (p sortFuncDoc) Len() int           { return len(p) }
+func (p sortFuncDoc) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p sortFuncDoc) Less(i, j int) bool { return p[i].Name < p[j].Name }
+
+func makeFuncDocs(m map[string]*ast.FuncDecl) []*FuncDoc {
+       d := make([]*FuncDoc, len(m))
+       i := 0
+       for _, f := range m {
+               doc := new(FuncDoc)
+               doc.Doc = CommentText(f.Doc)
+               f.Doc = nil // doc consumed - remove from ast.FuncDecl node
+               if f.Recv != nil {
+                       doc.Recv = f.Recv.List[0].Type
+               }
+               doc.Name = f.Name.Name
+               doc.Decl = f
+               d[i] = doc
+               i++
+       }
+       sort.Sort(sortFuncDoc(d))
+       return d
+}
+
+type methodSet map[string]*FuncDoc
+
+func (mset methodSet) add(m *FuncDoc) {
+       if mset[m.Name] == nil {
+               mset[m.Name] = m
+       }
+}
+
+func (mset methodSet) sortedList() []*FuncDoc {
+       list := make([]*FuncDoc, len(mset))
+       i := 0
+       for _, m := range mset {
+               list[i] = m
+               i++
+       }
+       sort.Sort(sortFuncDoc(list))
+       return list
+}
+
+type sortTypeDoc []*TypeDoc
+
+func (p sortTypeDoc) Len() int      { return len(p) }
+func (p sortTypeDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p sortTypeDoc) Less(i, j int) bool {
+       // sort by name
+       // pull blocks (name = "") up to top
+       // in original order
+       if ni, nj := p[i].Type.Name.Name, p[j].Type.Name.Name; ni != nj {
+               return ni < nj
+       }
+       return p[i].order < p[j].order
+}
+
+// NOTE(rsc): This would appear not to be correct for type ( )
+// blocks, but the doc extractor above has split them into
+// individual declarations.
+func (doc *docReader) makeTypeDocs(m map[string]*typeInfo) []*TypeDoc {
+       // TODO(gri) Consider computing the embedded method information
+       //           before calling makeTypeDocs. Then this function can
+       //           be single-phased again. Also, it might simplify some
+       //           of the logic.
+       //
+       // phase 1: associate collected declarations with TypeDocs
+       list := make([]*TypeDoc, len(m))
+       i := 0
+       for _, old := range m {
+               // all typeInfos should have a declaration associated with
+               // them after processing an entire package - be conservative
+               // and check
+               if decl := old.decl; decl != nil {
+                       typespec := decl.Specs[0].(*ast.TypeSpec)
+                       t := new(TypeDoc)
+                       doc := typespec.Doc
+                       typespec.Doc = nil // doc consumed - remove from ast.TypeSpec node
+                       if doc == nil {
+                               // no doc associated with the spec, use the declaration doc, if any
+                               doc = decl.Doc
+                       }
+                       decl.Doc = nil // doc consumed - remove from ast.Decl node
+                       t.Doc = CommentText(doc)
+                       t.Type = typespec
+                       t.Consts = makeValueDocs(old.values, token.CONST)
+                       t.Vars = makeValueDocs(old.values, token.VAR)
+                       t.Factories = makeFuncDocs(old.factories)
+                       t.methods = makeFuncDocs(old.methods)
+                       // The list of embedded types' methods is computed from the list
+                       // of embedded types, some of which may not have been processed
+                       // yet (i.e., their forward link is nil) - do this in a 2nd phase.
+                       // The final list of methods can only be computed after that -
+                       // do this in a 3rd phase.
+                       t.Decl = old.decl
+                       t.order = i
+                       old.forward = t // old has been processed
+                       list[i] = t
+                       i++
+               } else {
+                       // no corresponding type declaration found - move any associated
+                       // values, factory functions, and methods back to the top-level
+                       // so that they are not lost (this should only happen if a package
+                       // file containing the explicit type declaration is missing or if
+                       // an unqualified type name was used after a "." import)
+                       // 1) move values
+                       doc.values = append(doc.values, old.values...)
+                       // 2) move factory functions
+                       for name, f := range old.factories {
+                               doc.funcs[name] = f
+                       }
+                       // 3) move methods
+                       for name, f := range old.methods {
+                               // don't overwrite functions with the same name
+                               if _, found := doc.funcs[name]; !found {
+                                       doc.funcs[name] = f
+                               }
+                       }
+               }
+       }
+       list = list[0:i] // some types may have been ignored
+
+       // phase 2: collect embedded methods for each processed typeInfo
+       for _, old := range m {
+               if t := old.forward; t != nil {
+                       // old has been processed into t; collect embedded
+                       // methods for t from the list of processed embedded
+                       // types in old (and thus for which the methods are known)
+                       typ := t.Type
+                       if _, ok := typ.Type.(*ast.StructType); ok {
+                               // struct
+                               t.embedded = make(methodSet)
+                               collectEmbeddedMethods(t.embedded, old, typ.Name.Name)
+                       } else {
+                               // interface
+                               // TODO(gri) fix this
+                       }
+               }
+       }
+
+       // phase 3: compute final method set for each TypeDoc
+       for _, d := range list {
+               if len(d.embedded) > 0 {
+                       // there are embedded methods - exclude
+                       // the ones with names conflicting with
+                       // non-embedded methods
+                       mset := make(methodSet)
+                       // top-level methods have priority
+                       for _, m := range d.methods {
+                               mset.add(m)
+                       }
+                       // add non-conflicting embedded methods
+                       for _, m := range d.embedded {
+                               mset.add(m)
+                       }
+                       d.Methods = mset.sortedList()
+               } else {
+                       // no embedded methods
+                       d.Methods = d.methods
+               }
+       }
+
+       sort.Sort(sortTypeDoc(list))
+       return list
+}
+
+// collectEmbeddedMethods collects the embedded methods from all
+// processed embedded types found in info in mset. It considers
+// embedded types at the most shallow level first so that more
+// deeply nested embedded methods with conflicting names are
+// excluded.
+//
+func collectEmbeddedMethods(mset methodSet, info *typeInfo, recvTypeName string) {
+       for _, e := range info.embedded {
+               if e.typ.forward != nil { // == e was processed
+                       for _, m := range e.typ.forward.methods {
+                               mset.add(customizeRecv(m, e.ptr, recvTypeName))
+                       }
+                       collectEmbeddedMethods(mset, e.typ, recvTypeName)
+               }
+       }
+}
+
+func customizeRecv(m *FuncDoc, embeddedIsPtr bool, recvTypeName string) *FuncDoc {
+       if m == nil || m.Decl == nil || m.Decl.Recv == nil || len(m.Decl.Recv.List) != 1 {
+               return m // shouldn't happen, but be safe
+       }
+
+       // copy existing receiver field and set new type
+       // TODO(gri) is receiver type computation correct?
+       //           what about deeply nested embeddings?
+       newField := *m.Decl.Recv.List[0]
+       _, origRecvIsPtr := newField.Type.(*ast.StarExpr)
+       var typ ast.Expr = ast.NewIdent(recvTypeName)
+       if embeddedIsPtr || origRecvIsPtr {
+               typ = &ast.StarExpr{token.NoPos, typ}
+       }
+       newField.Type = typ
+
+       // copy existing receiver field list and set new receiver field
+       newFieldList := *m.Decl.Recv
+       newFieldList.List = []*ast.Field{&newField}
+
+       // copy existing function declaration and set new receiver field list
+       newFuncDecl := *m.Decl
+       newFuncDecl.Recv = &newFieldList
+
+       // copy existing function documentation and set new declaration
+       newM := *m
+       newM.Decl = &newFuncDecl
+       newM.Recv = typ
+
+       return &newM
+}
+
+func makeBugDocs(list []*ast.CommentGroup) []string {
+       d := make([]string, len(list))
+       for i, g := range list {
+               d[i] = CommentText(g)
+       }
+       return d
+}
+
+// newDoc returns the accumulated documentation for the package.
+//
+func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc {
+       p := new(PackageDoc)
+       p.PackageName = doc.pkgName
+       p.ImportPath = importpath
+       sort.Strings(filenames)
+       p.Filenames = filenames
+       p.Doc = CommentText(doc.doc)
+       // makeTypeDocs may extend the list of doc.values and
+       // doc.funcs and thus must be called before any other
+       // function consuming those lists
+       p.Types = doc.makeTypeDocs(doc.types)
+       p.Consts = makeValueDocs(doc.values, token.CONST)
+       p.Vars = makeValueDocs(doc.values, token.VAR)
+       p.Funcs = makeFuncDocs(doc.funcs)
+       p.Bugs = makeBugDocs(doc.bugs)
+       return p
+}