go/doc: better headscan

- scan all comments not just the package documentation
- declutter output so that false positives are more easily spotted
- count the number of headings to quickly see differences
- minor tweaks

R=golang-dev, r, r
CC=golang-dev
https://golang.org/cl/5450061

diff --git a/src/pkg/go/doc/comment.go b/src/pkg/go/doc/comment.go
index 44a047588d174d4ec1c871aeaf0aa80e437570ab..d7bb384ed03c5f5f7774989dac0665b34b782e1e 100644 (file)
--- a/src/pkg/go/doc/comment.go
+++ b/src/pkg/go/doc/comment.go
@@ -303,9 +303,8 @@ func heading(line []byte) []byte {
                return nil
        }
 
-       // allow ' for possessive 's only
-       b := line
-       for {
+       // allow "'" for possessive "'s" only
+       for b := line; ; {
                i := bytes.IndexRune(b, '\'')
                if i < 0 {
                        break
@@ -339,7 +338,7 @@ func heading(line []byte) []byte {
 func ToHTML(w io.Writer, s []byte, words map[string]string) {
        inpara := false
        lastWasBlank := false
-       lastNonblankWasHeading := false
+       lastWasHeading := false
 
        close := func() {
                if inpara {
@@ -389,10 +388,11 @@ func ToHTML(w io.Writer, s []byte, words map[string]string) {
                                emphasize(w, line, nil, false) // no nice text formatting
                        }
                        w.Write(html_endpre)
+                       lastWasHeading = false
                        continue
                }
 
-               if lastWasBlank && !lastNonblankWasHeading && i+2 < len(lines) &&
+               if lastWasBlank && !lastWasHeading && i+2 < len(lines) &&
                        isBlank(lines[i+1]) && !isBlank(lines[i+2]) && indentLen(lines[i+2]) == 0 {
                        // current line is non-blank, sourounded by blank lines
                        // and the next non-blank line is not indented: this
@@ -403,7 +403,7 @@ func ToHTML(w io.Writer, s []byte, words map[string]string) {
                                template.HTMLEscape(w, head)
                                w.Write(html_endh)
                                i += 2
-                               lastNonblankWasHeading = true
+                               lastWasHeading = true
                                continue
                        }
                }
@@ -411,7 +411,7 @@ func ToHTML(w io.Writer, s []byte, words map[string]string) {
                // open paragraph
                open()
                lastWasBlank = false
-               lastNonblankWasHeading = false
+               lastWasHeading = false
                emphasize(w, lines[i], words, true) // nice text formatting
                i++
        }

diff --git a/src/pkg/go/doc/comment_test.go b/src/pkg/go/doc/comment_test.go
index 9e77ae2cdea6be9e05061e330349cde712684383..870660ad62828b4a093d26ff3cf8a4a3567e4dd6 100644 (file)
--- a/src/pkg/go/doc/comment_test.go
+++ b/src/pkg/go/doc/comment_test.go
@@ -19,7 +19,7 @@ var headingTests = []struct {
        {"", false},
        {"section", false},
        {"A typical usage:", true},
-       {"δ is Greek", false}, // TODO: consider allowing this 
+       {"δ is Greek", false},
        {"Foo §", false},
        {"Fermat's Last Sentence", true},
        {"Fermat's", true},

diff --git a/src/pkg/go/doc/headscan.go b/src/pkg/go/doc/headscan.go
index 95953b3bdcbe008f738b9231b032611a9dd7e668..83f24627c956fcfebad171cafc3e620d8c366266 100644 (file)
--- a/src/pkg/go/doc/headscan.go
+++ b/src/pkg/go/doc/headscan.go
@@ -1,53 +1,111 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+       The headscan command extracts comment headings from package files;
+       it is used to detect false positives which may require an adjustment
+       to the comment formatting heuristics in comment.go.
+
+       Usage: headscan [-root root_directory]
+
+       By default, the $GOROOT/src directory is scanned.
+*/
 package main
 
 import (
        "bytes"
        "flag"
+       "fmt"
        "go/doc"
        "go/parser"
        "go/token"
-       "log"
        "os"
        "path/filepath"
+       "runtime"
        "strings"
 )
 
+var (
+       root    = flag.String("root", filepath.Join(runtime.GOROOT(), "src"), "root of filesystem tree to scan")
+       verbose = flag.Bool("v", false, "verbose mode")
+)
+
+const (
+       html_h    = "<h3>"
+       html_endh = "</h3>\n"
+)
+
 func isGoFile(fi os.FileInfo) bool {
        return strings.HasSuffix(fi.Name(), ".go") &&
                !strings.HasSuffix(fi.Name(), "_test.go")
 }
 
+func appendHeadings(list []string, comment string) []string {
+       var buf bytes.Buffer
+       doc.ToHTML(&buf, []byte(comment), nil)
+       for s := buf.String(); ; {
+               i := strings.Index(s, html_h)
+               if i < 0 {
+                       break
+               }
+               i += len(html_h)
+               j := strings.Index(s, html_endh)
+               if j < 0 {
+                       list = append(list, s[i:]) // incorrect HTML
+                       break
+               }
+               list = append(list, s[i:j])
+               s = s[j+len(html_endh):]
+       }
+       return list
+}
+
 func main() {
-       fset := token.NewFileSet()
-       rootDir := flag.String("root", "./", "root of filesystem tree to scan")
        flag.Parse()
-       err := filepath.Walk(*rootDir, func(path string, fi os.FileInfo, err error) error {
+       fset := token.NewFileSet()
+       nheadings := 0
+       err := filepath.Walk(*root, func(path string, fi os.FileInfo, err error) error {
                if !fi.IsDir() {
                        return nil
                }
                pkgs, err := parser.ParseDir(fset, path, isGoFile, parser.ParseComments)
                if err != nil {
-                       log.Println(path, err)
+                       if *verbose {
+                               fmt.Fprintln(os.Stderr, err)
+                       }
                        return nil
                }
                for _, pkg := range pkgs {
                        d := doc.NewPackageDoc(pkg, path)
-                       buf := new(bytes.Buffer)
-                       doc.ToHTML(buf, []byte(d.Doc), nil)
-                       b := buf.Bytes()
-                       for {
-                               i := bytes.Index(b, []byte("<h3>"))
-                               if i == -1 {
-                                       break
+                       list := appendHeadings(nil, d.Doc)
+                       for _, d := range d.Consts {
+                               list = appendHeadings(list, d.Doc)
+                       }
+                       for _, d := range d.Types {
+                               list = appendHeadings(list, d.Doc)
+                       }
+                       for _, d := range d.Vars {
+                               list = appendHeadings(list, d.Doc)
+                       }
+                       for _, d := range d.Funcs {
+                               list = appendHeadings(list, d.Doc)
+                       }
+                       if len(list) > 0 {
+                               // directories may contain multiple packages;
+                               // print path and package name
+                               fmt.Printf("%s (package %s)\n", path, pkg.Name)
+                               for _, h := range list {
+                                       fmt.Printf("\t%s\n", h)
                                }
-                               line := bytes.SplitN(b[i:], []byte("\n"), 2)[0]
-                               log.Printf("%s: %s", path, line)
-                               b = b[i+len(line):]
+                               nheadings += len(list)
                        }
                }
                return nil
        })
        if err != nil {
-               log.Fatal(err)
+               fmt.Fprintln(os.Stderr, err)
+               os.Exit(1)
        }
+       fmt.Println(nheadings, "headings found")
 }