text/scanner: add examples

Added examples for use of Mode, Whitespace, and IsIdentRune properties.

Fixes #23768

Change-Id: I2528e14fde63a4476f3c25510bf0c5b73f38ba5d
Reviewed-on: https://go-review.googlesource.com/93199
Run-TryBot: Robert Griesemer <gri@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

diff --git a/src/text/scanner/example_test.go b/src/text/scanner/example_test.go
index 97e22a98f4155e02b80853978af25723b53d4331..5e8c3fbda4ff88abe0551b807fc6b8d0cce61a60 100644 (file)
--- a/src/text/scanner/example_test.go
+++ b/src/text/scanner/example_test.go
@@ -1,4 +1,4 @@
-// Copyright 2015 The Go Authors. All rights reserved.
+// Copyright 2018 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -8,6 +8,7 @@ import (
        "fmt"
        "strings"
        "text/scanner"
+       "unicode"
 )
 
 func Example() {
@@ -16,6 +17,7 @@ func Example() {
 if a > 10 {
        someParsable = text
 }`
+
        var s scanner.Scanner
        s.Init(strings.NewReader(src))
        s.Filename = "example"
@@ -34,3 +36,105 @@ if a > 10 {
        // example:4:17: text
        // example:5:1: }
 }
+
+func Example_isIdentRune() {
+       const src = "%var1 var2%"
+
+       var s scanner.Scanner
+       s.Init(strings.NewReader(src))
+       s.Filename = "default"
+
+       for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+               fmt.Printf("%s: %s\n", s.Position, s.TokenText())
+       }
+
+       fmt.Println()
+       s.Init(strings.NewReader(src))
+       s.Filename = "percent"
+
+       // treat leading '%' as part of an identifier
+       s.IsIdentRune = func(ch rune, i int) bool {
+               return ch == '%' && i == 0 || unicode.IsLetter(ch) || unicode.IsDigit(ch) && i > 0
+       }
+
+       for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+               fmt.Printf("%s: %s\n", s.Position, s.TokenText())
+       }
+
+       // Output:
+       // default:1:1: %
+       // default:1:2: var1
+       // default:1:7: var2
+       // default:1:11: %
+       //
+       // percent:1:1: %var1
+       // percent:1:7: var2
+       // percent:1:11: %
+}
+
+func Example_mode() {
+       const src = `
+    // Comment begins at column 5.
+
+This line should not be included in the output.
+
+/*
+This multiline comment
+should be extracted in
+its entirety.
+*/
+`
+
+       var s scanner.Scanner
+       s.Init(strings.NewReader(src))
+       s.Filename = "comments"
+       s.Mode ^= scanner.SkipComments // don't skip comments
+
+       for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+               txt := s.TokenText()
+               if strings.HasPrefix(txt, "//") || strings.HasPrefix(txt, "/*") {
+                       fmt.Printf("%s: %s\n", s.Position, txt)
+               }
+       }
+
+       // Output:
+       // comments:2:5: // Comment begins at column 5.
+       // comments:6:1: /*
+       // This multiline comment
+       // should be extracted in
+       // its entirety.
+       // */
+}
+
+func Example_whitespace() {
+       // tab-separated values
+       const src = `aa ab      ac      ad
+ba     bb      bc      bd
+ca     cb      cc      cd
+da     db      dc      dd`
+
+       var (
+               col, row int
+               s        scanner.Scanner
+               tsv      [4][4]string // large enough for example above
+       )
+       s.Init(strings.NewReader(src))
+       s.Whitespace ^= 1<<'\t' | 1<<'\n' // don't skip tabs and new lines
+
+       for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+               switch tok {
+               case '\n':
+                       row++
+                       col = 0
+               case '\t':
+                       col++
+               default:
+                       tsv[row][col] = s.TokenText()
+               }
+       }
+
+       fmt.Print(tsv)
+
+       // Output:
+       // [[aa ab ac ad] [ba bb bc bd] [ca cb cc cd] [da db dc dd]]
+}