text/scanner: improve documentation

LGTM=r
R=r
CC=golang-codereviews
https://golang.org/cl/123390043

diff --git a/src/pkg/text/scanner/scanner.go b/src/pkg/text/scanner/scanner.go
index 25b2c9e02afb79a6e892b5a8b4a278e4448b824d..5199ee4fc7dc4c4e9afbd4c84454dee9a13cb753 100644 (file)
--- a/src/pkg/text/scanner/scanner.go
+++ b/src/pkg/text/scanner/scanner.go
@@ -66,6 +66,12 @@ func (pos Position) String() string {
 //
 //     ScanIdents | ScanInts | SkipComments
 //
+// With the exceptions of comments, which are skipped if SkipComments is
+// set, unrecognized tokens are not ignored. Instead, the scanner simply
+// returns the respective individual characters (or possibly sub-tokens).
+// For instance, if the mode is ScanIdents (not ScanStrings), the string
+// "foo" is scanned as the token sequence '"' Ident '"'.
+//
 const (
        ScanIdents     = 1 << -Ident
        ScanInts       = 1 << -Int